From b5f47a074dbc136f54327eb4a7c1122f1c7829a0 Mon Sep 17 00:00:00 2001 From: Christopher Date: Thu, 23 Apr 2026 12:16:41 +1000 Subject: [PATCH 1/7] feat(config): add hooks.pre_run for pre-eval environment injection Adds a hooks.pre_run field to both agentv.config.ts (as hooks.preRun) and .agentv/config.yaml (as hooks.pre_run) that runs a shell command before an eval starts and injects its exported env vars into process.env. - New hooks.ts utility: parseEnvOutput + runPreRunHook - Parses both `export KEY="value"` and `KEY=value` stdout formats - Only injects keys not already set in process.env (existing env wins) - Forwards stderr to process.stderr; non-zero exit aborts the eval - Wired into runEvalCommand before normalizeOptions so secrets are available for all subsequent config and env lookups - JSON schema updated for .agentv/config.yaml IDE autocomplete Closes #1149 Co-Authored-By: Claude Sonnet 4.6 --- apps/cli/src/commands/eval/run-eval.ts | 8 ++ docs/plans/pre-run-hook.md | 39 ++++++ packages/core/src/evaluation/config.ts | 14 ++ packages/core/src/evaluation/hooks.ts | 123 ++++++++++++++++++ .../src/evaluation/loaders/config-loader.ts | 35 +++++ packages/core/src/index.ts | 1 + packages/core/test/evaluation/hooks.test.ts | 64 +++++++++ .../references/config-schema.json | 12 ++ 8 files changed, 296 insertions(+) create mode 100644 docs/plans/pre-run-hook.md create mode 100644 packages/core/src/evaluation/hooks.ts create mode 100644 packages/core/test/evaluation/hooks.test.ts diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 6061d5448..322cda695 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -22,6 +22,7 @@ import { loadTestSuite, loadTsConfig, resolveTargetDefinition, + runPreRunHook, shouldEnableCache, shouldSkipCacheForTemperature, subscribeToCodexLogEntries, @@ -992,6 +993,13 @@ export async function runEvalCommand( }); } + // Run pre-run hook (YAML config takes precedence; TS config is fallback) + // Hooks run before normalizeOptions so secrets are available for env interpolation + const preRunCommand = yamlConfig?.hooks?.pre_run ?? config?.hooks?.preRun; + if (preRunCommand) { + runPreRunHook(preRunCommand); + } + let options = normalizeOptions(input.rawOptions, config, yamlConfig?.execution); if (!process.env.AGENTV_EXPERIMENT) { process.env.AGENTV_EXPERIMENT = normalizeExperimentName(options.experiment); diff --git a/docs/plans/pre-run-hook.md b/docs/plans/pre-run-hook.md new file mode 100644 index 000000000..a469f35ff --- /dev/null +++ b/docs/plans/pre-run-hook.md @@ -0,0 +1,39 @@ +# Plan: hooks.pre_run — pre-eval environment injection + +Issue: #1149 + +## Problem + +Users who fetch secrets at runtime (e.g. from Azure Key Vault, AWS Secrets Manager) must wrap the +`agentv` CLI in a project-level script. A generic `hooks.pre_run` config option removes this need. + +## Implementation + +### Files changed + +| File | Change | +|------|--------| +| `packages/core/src/evaluation/hooks.ts` | New — `parseEnvOutput()` and `runPreRunHook()` | +| `packages/core/src/evaluation/config.ts` | Add `hooks.preRun` to Zod schema + TS type | +| `packages/core/src/evaluation/loaders/config-loader.ts` | Add `HooksConfig` type + `parseHooksConfig()` + wire into `loadConfig()` | +| `packages/core/src/index.ts` | Export `runPreRunHook`, `parseEnvOutput` | +| `apps/cli/src/commands/eval/run-eval.ts` | Import `runPreRunHook`; call before `normalizeOptions` | +| `plugins/agentv-dev/skills/agentv-eval-writer/references/config-schema.json` | Add `hooks.pre_run` to JSON schema | +| `packages/core/test/evaluation/hooks.test.ts` | New — unit tests for `parseEnvOutput` | + +### Precedence + +YAML config (`hooks.pre_run`) takes precedence over TS config (`hooks.preRun`). This matches the +existing pattern for other settings. + +### Env var injection rules + +- Parses `export KEY="value"` (shell export) and `KEY=value` (dotenv) from stdout +- Existing env vars are NOT overwritten — process.env always wins +- Non-zero exit throws, aborting the eval +- Stderr forwarded to process.stderr so users see hook output + +## Docs to update before merge + +- `apps/web/src/content/docs/` — add `hooks.pre_run` to the config reference page +- Delete this plan file diff --git a/packages/core/src/evaluation/config.ts b/packages/core/src/evaluation/config.ts index 1f2238f39..c06f45fb2 100644 --- a/packages/core/src/evaluation/config.ts +++ b/packages/core/src/evaluation/config.ts @@ -78,6 +78,20 @@ const AgentVConfigSchema = z.object({ maxDurationMs: z.number().int().min(0).optional(), }) .optional(), + + /** Lifecycle hooks */ + hooks: z + .object({ + /** + * Shell command to run before the eval starts. + * stdout is parsed for env var exports (`KEY=value` or `export KEY="value"`) + * and injected into process.env. Keys already set in the environment are + * not overwritten — existing env always takes priority. + * stderr is forwarded to the user. Non-zero exit aborts the eval. + */ + preRun: z.string().optional(), + }) + .optional(), }); /** diff --git a/packages/core/src/evaluation/hooks.ts b/packages/core/src/evaluation/hooks.ts new file mode 100644 index 000000000..c36463f3b --- /dev/null +++ b/packages/core/src/evaluation/hooks.ts @@ -0,0 +1,123 @@ +/** + * Pre-run hook execution for AgentV. + * + * Runs a shell command before an eval starts and injects exported environment + * variables into the current process. This lets projects fetch secrets at + * runtime (e.g. from a vault) without needing a wrapper script. + * + * ## How it works + * + * 1. The command is run via `sh -c` (or `cmd /c` on Windows). + * 2. stdout is captured and parsed for env var exports. + * 3. stderr is forwarded to the process stderr so the user sees output. + * 4. Non-zero exit aborts the eval with a clear error. + * 5. Parsed keys are injected into `process.env` — only for keys not already + * set, so existing env always wins. + * + * ## Supported output formats + * + * Both shell-export and dotenv formats are accepted: + * export KEY="value" (shell export — quotes optional) + * KEY=value (dotenv — no export prefix) + * + * Lines that don't match either pattern are silently ignored. + * + * @module + */ + +import { spawnSync } from 'node:child_process'; + +const ANSI_YELLOW = ''; +const ANSI_RESET = ''; + +/** + * Parse env var lines from hook stdout. + * + * Accepts: + * export KEY="value" → { KEY: "value" } + * export KEY=value → { KEY: "value" } + * KEY=value → { KEY: "value" } + * + * Strips surrounding single or double quotes from values. + * Skips lines with empty keys or values that look like shell syntax. + */ +export function parseEnvOutput(stdout: string): Record { + const result: Record = {}; + + for (const line of stdout.split('\n')) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) continue; + + // Match: [export ]KEY=value + const match = trimmed.match(/^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$/); + if (!match) continue; + + const key = match[1]; + let value = match[2]; + + // Strip surrounding quotes (single or double) + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + + if (key) { + result[key] = value; + } + } + + return result; +} + +/** + * Run the pre_run hook command and inject exported env vars into process.env. + * + * - Runs via shell (`sh -c` on POSIX, `cmd /c` on Windows) + * - Captured stdout is parsed for env vars; stderr is forwarded to process.stderr + * - Non-zero exit throws an Error with the command and exit code + * - Keys already set in process.env are NOT overwritten + * + * @param command Shell command string to execute + */ +export function runPreRunHook(command: string): void { + const isWindows = process.platform === 'win32'; + const shell = isWindows ? 'cmd' : 'sh'; + const shellFlag = isWindows ? '/c' : '-c'; + + console.log(`${ANSI_YELLOW}Running pre-run hook: ${command}${ANSI_RESET}`); + + const result = spawnSync(shell, [shellFlag, command], { + encoding: 'utf8', + // Do not inherit stdio — capture stdout for parsing, forward stderr manually + stdio: ['ignore', 'pipe', 'pipe'], + }); + + // Forward stderr so the user can see hook output (warnings, progress, etc.) + if (result.stderr) { + process.stderr.write(result.stderr); + } + + if (result.error) { + throw new Error(`Pre-run hook failed to start: ${result.error.message}`); + } + + if (result.status !== 0) { + throw new Error(`Pre-run hook exited with code ${result.status ?? 'unknown'}: ${command}`); + } + + const vars = parseEnvOutput(result.stdout ?? ''); + let injected = 0; + + for (const [key, value] of Object.entries(vars)) { + if (process.env[key] === undefined) { + process.env[key] = value; + injected++; + } + } + + if (injected > 0) { + console.log(`Pre-run hook injected ${injected} environment variable(s).`); + } +} diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index 377222123..c204f9360 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -43,6 +43,11 @@ export type ResultsExportConfig = { readonly branch_prefix?: string; }; +export type HooksConfig = { + /** Shell command to run before the eval starts. stdout is parsed for env var exports. */ + readonly pre_run?: string; +}; + export type AgentVConfig = { readonly required_version?: string; readonly eval_patterns?: readonly string[]; @@ -50,6 +55,7 @@ export type AgentVConfig = { readonly results?: { readonly export?: ResultsExportConfig; }; + readonly hooks?: HooksConfig; }; /** @@ -102,12 +108,14 @@ export async function loadConfig( configPath, ); const results = parseResultsConfig((parsed as Record).results, configPath); + const hooks = parseHooksConfig((parsed as Record).hooks, configPath); return { required_version: requiredVersion as string | undefined, eval_patterns: evalPatterns as readonly string[] | undefined, execution: executionDefaults, results, + ...(hooks && { hooks }), }; } catch (error) { logWarning( @@ -623,6 +631,33 @@ export function parseResultsExportConfig( }; } +/** + * Parse the `hooks` block from .agentv/config.yaml. + * Currently supports `pre_run` only. + */ +export function parseHooksConfig(raw: unknown, configPath: string): HooksConfig | undefined { + if (raw === undefined || raw === null) { + return undefined; + } + if (typeof raw !== 'object' || Array.isArray(raw)) { + logWarning(`Invalid hooks in ${configPath}, expected object`); + return undefined; + } + + const obj = raw as Record; + + const preRun = obj.pre_run; + if (preRun !== undefined) { + if (typeof preRun !== 'string' || preRun.trim().length === 0) { + logWarning(`Invalid hooks.pre_run in ${configPath}, expected non-empty string`); + return undefined; + } + return { pre_run: preRun.trim() }; + } + + return undefined; +} + function logWarning(message: string): void { console.warn(`${ANSI_YELLOW}Warning: ${message}${ANSI_RESET}`); } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 3e9a475da..da8887e8b 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -129,6 +129,7 @@ export { } from './evaluation/graders/assertions.js'; export { discoverGraders } from './evaluation/registry/grader-discovery.js'; export { RunBudgetTracker } from './evaluation/run-budget-tracker.js'; +export { runPreRunHook, parseEnvOutput } from './evaluation/hooks.js'; // Import pipeline export * from './import/index.js'; diff --git a/packages/core/test/evaluation/hooks.test.ts b/packages/core/test/evaluation/hooks.test.ts new file mode 100644 index 000000000..802a43631 --- /dev/null +++ b/packages/core/test/evaluation/hooks.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from 'bun:test'; + +import { parseEnvOutput } from '../../src/evaluation/hooks.js'; + +describe('parseEnvOutput', () => { + it('parses dotenv KEY=value lines', () => { + expect(parseEnvOutput('FOO=bar\nBAZ=qux')).toEqual({ FOO: 'bar', BAZ: 'qux' }); + }); + + it('parses export KEY="value" lines (double quotes)', () => { + expect(parseEnvOutput('export FOO="bar"')).toEqual({ FOO: 'bar' }); + }); + + it("parses export KEY='value' lines (single quotes)", () => { + expect(parseEnvOutput("export FOO='bar'")).toEqual({ FOO: 'bar' }); + }); + + it('parses export KEY=value without quotes', () => { + expect(parseEnvOutput('export FOO=bar')).toEqual({ FOO: 'bar' }); + }); + + it('allows values containing equals signs', () => { + expect(parseEnvOutput('FOO=a=b=c')).toEqual({ FOO: 'a=b=c' }); + }); + + it('handles empty values', () => { + expect(parseEnvOutput('FOO=')).toEqual({ FOO: '' }); + }); + + it('ignores comment lines', () => { + expect(parseEnvOutput('# This is a comment\nFOO=bar')).toEqual({ FOO: 'bar' }); + }); + + it('ignores blank lines', () => { + expect(parseEnvOutput('\n\nFOO=bar\n\n')).toEqual({ FOO: 'bar' }); + }); + + it('ignores lines that are not env var assignments', () => { + expect(parseEnvOutput('not-valid\nFOO=bar\necho hello')).toEqual({ FOO: 'bar' }); + }); + + it('parses multiple mixed-format lines', () => { + const input = [ + 'export KEY1="value1"', + "export KEY2='value2'", + 'KEY3=value3', + 'export KEY4=value4', + ].join('\n'); + expect(parseEnvOutput(input)).toEqual({ + KEY1: 'value1', + KEY2: 'value2', + KEY3: 'value3', + KEY4: 'value4', + }); + }); + + it('returns empty object for empty stdout', () => { + expect(parseEnvOutput('')).toEqual({}); + }); + + it('accepts keys with underscores and digits', () => { + expect(parseEnvOutput('MY_KEY_123=hello')).toEqual({ MY_KEY_123: 'hello' }); + }); +}); diff --git a/plugins/agentv-dev/skills/agentv-eval-writer/references/config-schema.json b/plugins/agentv-dev/skills/agentv-eval-writer/references/config-schema.json index bfdf737de..58417af73 100644 --- a/plugins/agentv-dev/skills/agentv-eval-writer/references/config-schema.json +++ b/plugins/agentv-dev/skills/agentv-eval-writer/references/config-schema.json @@ -44,6 +44,18 @@ } }, "additionalProperties": false + }, + "hooks": { + "type": "object", + "description": "Lifecycle hooks that run at specific points in the eval lifecycle.", + "properties": { + "pre_run": { + "type": "string", + "description": "Shell command to run before the eval starts. stdout is parsed for env var exports (KEY=value or export KEY=\"value\") and injected into process.env. Keys already set in the environment are not overwritten. stderr is forwarded to the user. Non-zero exit aborts the eval.", + "examples": ["bun scripts/load-secrets.ts", "eval $(aws ssm get-parameters-by-path ...)"] + } + }, + "additionalProperties": false } }, "required": ["$schema"], From 1d481f1534535463d4578d3d99cc94c6a31992ab Mon Sep 17 00:00:00 2001 From: Christopher Date: Thu, 23 Apr 2026 12:24:44 +1000 Subject: [PATCH 2/7] docs(agents): replace in-progress label with project board status for claim tracking Co-Authored-By: Claude Sonnet 4.6 --- AGENTS.md | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 13c3cd0cc..6045f8a46 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -406,7 +406,7 @@ Types: `feat`, `fix`, `docs`, `style`, `refactor`, `test`, `chore` When working on a GitHub issue, **ALWAYS** follow this workflow: -1. **Claim the issue** — prevents other agents from duplicating work: +1. **Claim the issue** — prevents other agents from duplicating work by stamping Agent ID and setting status on the project board: ```bash # Load AGENT_ID from .env; if not set, ask the user or default to - # Harness = the coding tool (claude-code, opencode, codex-cli, cursor, etc.) @@ -419,15 +419,13 @@ When working on a GitHub issue, **ALWAYS** follow this workflow: echo "AGENT_ID is not set. Ask the user for an agent identifier, or default to devbox2-codex in this environment (otherwise use -)." fi - # Check if already claimed - gh issue view --json labels --jq '.labels[].name' | grep -q "in-progress" && echo "SKIP — already claimed" && exit 1 - - # Claim it — label + project roadmap status - gh issue edit --add-label "in-progress" + # Check if already claimed via project board status + ITEM_ID=$(gh project item-list 1 --owner EntityProcess --format json | jq -r '.items[] | select(.content.number == and .content.repository == "EntityProcess/agentv") | .id') + CURRENT_STATUS=$(gh project item-list 1 --owner EntityProcess --format json | jq -r '.items[] | select(.content.number == and .content.repository == "EntityProcess/agentv") | .status') + [ "$CURRENT_STATUS" = "In Progress" ] && echo "SKIP — already claimed" && exit 1 # Update project roadmap: ensure the issue is on the AgentV OSS board, - # then set status to "In progress" and stamp Agent ID - ITEM_ID=$(gh project item-list 1 --owner EntityProcess --format json | jq -r '.items[] | select(.content.number == and .content.repository == "EntityProcess/agentv") | .id') + # then set status to "In Progress" and stamp Agent ID if [ -z "$ITEM_ID" ] || [ "$ITEM_ID" = "null" ]; then ITEM_ID=$(gh project item-add 1 --owner EntityProcess --url "https://github.com/EntityProcess/agentv/issues/" --format json | jq -r '.id') fi @@ -436,7 +434,7 @@ When working on a GitHub issue, **ALWAYS** follow this workflow: gh project item-edit --project-id PVT_kwDOAIbbRc4BSmjF --id "$ITEM_ID" --field-id PVTF_lADOAIbbRc4BSmjFzhAHSnk --text "$AGENT_ID" fi ``` - If the issue has the `in-progress` label, **do not work on it** — pick a different issue. + If the issue has project board status "In Progress", **do not work on it** — pick a different issue. 2. **Update local `main` to the latest `origin/main`** before branching: ```bash @@ -481,17 +479,14 @@ When working on a GitHub issue, **ALWAYS** follow this workflow: - Remove the local worktree created for the issue - Confirm the primary checkout is back on an up-to-date `main` -The `in-progress` label stays on the issue until the PR is merged and the issue is closed. Do not remove it manually. - **IMPORTANT:** Never push directly to `main`. Always use branches and PRs. ### Tracker Conventions -- The roadmap project is the source of truth for prioritization. +- The roadmap project is the source of truth for prioritization and claim status — use it, not labels. - Issues in the roadmap are prioritized; issues outside it are not. - `bug` marks defects. - Issues without `bug` are non-bug work by default. -- `in-progress` marks an issue as claimed by an agent — do not start work on it. - `core`, `wui`, and `tui` are area labels. - Keep issue bodies focused on the handoff contract: objective, design latitude, acceptance signals, non-goals, and related links. - Do not put priority metadata in issue bodies. From 7e75597f3fa0140dafe0a9b28118650fb1c0b91c Mon Sep 17 00:00:00 2001 From: Christopher Date: Thu, 23 Apr 2026 12:30:59 +1000 Subject: [PATCH 3/7] docs(agents): make code review step optional for focused changes Co-Authored-By: Claude Sonnet 4.6 --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 6045f8a46..a8ec14380 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -466,7 +466,7 @@ When working on a GitHub issue, **ALWAYS** follow this workflow: 1. Run unit tests. 2. Execute every test plan item from the issue/PR checklist, mark each `[x]`, and paste CLI output as evidence. 3. Manual red/green UAT with before/after evidence. - 4. **After e2e passes**, spawn a final subagent code review pass and address or call out any findings. Do NOT run the code review before e2e — if e2e fails you'll need to fix it first, which invalidates the review. + 4. **After e2e passes**, spawn a final subagent code review pass and address or call out any findings — **unless the change is focused** (single-responsibility, well-tested, no architectural impact), in which case this step may be skipped. Do NOT run the code review before e2e — if e2e fails you'll need to fix it first, which invalidates the review. 5. CI pipeline passes (all checks green). 6. No merge conflicts with `main`. From 74adef05fda81e7eabf3e62b8efe8949be9386bd Mon Sep 17 00:00:00 2001 From: Christopher Date: Thu, 23 Apr 2026 12:41:05 +1000 Subject: [PATCH 4/7] fix(cli): return schema-valid grader response from --dry-run mock --dry-run previously returned '{"answer":"Mock dry-run response"}' which caused LLM graders to fail with 'Required: score' parse errors after 3 attempts. The mock response now satisfies all three grader schemas (freeform, rubric, score-range) so --dry-run works end-to-end including grader plumbing without real LLM calls. Co-Authored-By: Claude Sonnet 4.6 --- AGENTS.md | 2 +- apps/cli/src/commands/eval/run-eval.ts | 6 ++- apps/cli/src/commands/eval/targets.ts | 15 +++++- packages/core/src/evaluation/graders/index.ts | 1 + .../core/src/evaluation/graders/llm-grader.ts | 2 +- .../graders/dry-run-mock-response.test.ts | 52 +++++++++++++++++++ 6 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 packages/core/test/evaluation/graders/dry-run-mock-response.test.ts diff --git a/AGENTS.md b/AGENTS.md index a8ec14380..70f523fd3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -334,7 +334,7 @@ Unit tests alone are insufficient for grader changes. After implementing or modi 4. **Update baseline files** if output format changes (e.g., type name renames). Baseline files live alongside eval YAML files as `*.baseline.jsonl` and contain expected `scores[].type` values. There are 30+ baseline files across `examples/`. -5. **Note:** `--dry-run` returns schema-valid mock responses (`{}` as output, zeroed `tokenUsage`). Built-in graders will not crash, but scores are meaningless. Use it for testing harness flow, not grader logic. +5. **Note:** `--dry-run` returns schema-valid mock responses for both agent output and grader evaluation (score=1, empty assertions/checks). Built-in LLM graders run without parse errors but scores are meaningless. Use it for end-to-end harness testing including grader plumbing. ### Completing Work — E2E Checklist diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 322cda695..3d60dbee2 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -590,7 +590,11 @@ async function prepareFileMetadata(params: { name: `${targetDefinition.name}-dry-run`, graderTarget: undefined, config: { - response: '{"answer":"Mock dry-run response"}', + // Schema-valid grader response so --dry-run works end-to-end with LLM graders. + // Satisfies freeform (score), rubric (checks, overall_reasoning), and + // score-range (checks) schemas without real LLM calls. + response: + '{"score":1,"assertions":[],"checks":[],"overall_reasoning":"dry-run mock"}', delayMs: options.dryRunDelay, delayMinMs: options.dryRunDelayMin, delayMaxMs: options.dryRunDelayMax, diff --git a/apps/cli/src/commands/eval/targets.ts b/apps/cli/src/commands/eval/targets.ts index ad902f0a8..25730cc61 100644 --- a/apps/cli/src/commands/eval/targets.ts +++ b/apps/cli/src/commands/eval/targets.ts @@ -13,6 +13,17 @@ const ANSI_YELLOW = '\u001b[33m'; const ANSI_RED = '\u001b[31m'; const ANSI_RESET = '\u001b[0m'; +/** + * Dry-run mock response: satisfies all LLM grader schemas (freeform, rubric, score-range) + * so that --dry-run works end-to-end including graders without real LLM calls. + * + * - freeformEvaluationSchema: "score" (required), "assertions" (optional) + * - rubricEvaluationSchema: "checks" (required), "overall_reasoning" (required) + * - scoreRangeEvaluationSchema: "checks" (required), "overall_reasoning" (optional) + */ +const DRY_RUN_MOCK_RESPONSE = + '{"score":1,"assertions":[],"checks":[],"overall_reasoning":"dry-run mock"}'; + function isTTY(): boolean { return process.stdout.isTTY ?? false; } @@ -183,7 +194,7 @@ export async function selectTarget(options: TargetSelectionOptions): Promise { + it('is valid JSON', () => { + expect(() => JSON.parse(DRY_RUN_MOCK_RESPONSE)).not.toThrow(); + }); + + it('satisfies freeformEvaluationSchema (requires score)', () => { + const parsed = JSON.parse(DRY_RUN_MOCK_RESPONSE); + const result = freeformEvaluationSchema.safeParse(parsed); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.score).toBe(1); + } + }); + + it('satisfies rubricEvaluationSchema (requires checks and overall_reasoning)', () => { + const parsed = JSON.parse(DRY_RUN_MOCK_RESPONSE); + const result = rubricEvaluationSchema.safeParse(parsed); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.checks).toEqual([]); + expect(result.data.overall_reasoning).toBe('dry-run mock'); + } + }); + + it('satisfies scoreRangeEvaluationSchema (requires checks, optional overall_reasoning)', () => { + const parsed = JSON.parse(DRY_RUN_MOCK_RESPONSE); + const result = scoreRangeEvaluationSchema.safeParse(parsed); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.checks).toEqual([]); + } + }); +}); From 0e31f238c6618721ffcd71250bd64917a9ec353b Mon Sep 17 00:00:00 2001 From: Christopher Date: Thu, 23 Apr 2026 12:42:15 +1000 Subject: [PATCH 5/7] style(cli): fix biome formatting for dry-run mock response Co-Authored-By: Claude Sonnet 4.6 --- apps/cli/src/commands/eval/run-eval.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 3d60dbee2..53ad4803a 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -591,10 +591,8 @@ async function prepareFileMetadata(params: { graderTarget: undefined, config: { // Schema-valid grader response so --dry-run works end-to-end with LLM graders. - // Satisfies freeform (score), rubric (checks, overall_reasoning), and - // score-range (checks) schemas without real LLM calls. - response: - '{"score":1,"assertions":[],"checks":[],"overall_reasoning":"dry-run mock"}', + // Satisfies freeform (score), rubric (checks, overall_reasoning), and score-range (checks) without real LLM calls. + response: '{"score":1,"assertions":[],"checks":[],"overall_reasoning":"dry-run mock"}', delayMs: options.dryRunDelay, delayMinMs: options.dryRunDelayMin, delayMaxMs: options.dryRunDelayMax, From e5f5849407ef1dd411447ec8e0e6517da136189e Mon Sep 17 00:00:00 2001 From: Christopher Date: Thu, 23 Apr 2026 13:10:52 +1000 Subject: [PATCH 6/7] feat(config): rename pre_run to before_session, fire hook at CLI entrypoint Hook now runs once per agentv invocation (not once per eval run), covering all commands including interactive mode. Equivalent to the project-level wrapper script pattern from the end user's perspective. Co-Authored-By: Claude Sonnet 4.6 --- apps/cli/src/commands/eval/run-eval.ts | 7 ------- apps/cli/src/index.ts | 14 +++++++++++++ packages/core/src/evaluation/config.ts | 6 +++--- packages/core/src/evaluation/hooks.ts | 20 ++++++++++--------- .../src/evaluation/loaders/config-loader.ts | 16 +++++++-------- packages/core/src/index.ts | 2 +- .../references/config-schema.json | 6 +++--- 7 files changed, 40 insertions(+), 31 deletions(-) diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 53ad4803a..f428a2791 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -22,7 +22,6 @@ import { loadTestSuite, loadTsConfig, resolveTargetDefinition, - runPreRunHook, shouldEnableCache, shouldSkipCacheForTemperature, subscribeToCodexLogEntries, @@ -995,12 +994,6 @@ export async function runEvalCommand( }); } - // Run pre-run hook (YAML config takes precedence; TS config is fallback) - // Hooks run before normalizeOptions so secrets are available for env interpolation - const preRunCommand = yamlConfig?.hooks?.pre_run ?? config?.hooks?.preRun; - if (preRunCommand) { - runPreRunHook(preRunCommand); - } let options = normalizeOptions(input.rawOptions, config, yamlConfig?.execution); if (!process.env.AGENTV_EXPERIMENT) { diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index 59500586f..d55ce5863 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -1,4 +1,7 @@ +import path from 'node:path'; import { binary, run, subcommands } from 'cmd-ts'; +import { loadConfig, runBeforeSessionHook } from '@agentv/core'; +import { findRepoRoot } from './commands/eval/shared.js'; import packageJson from '../package.json' with { type: 'json' }; import { compareCommand } from './commands/compare/index.js'; @@ -130,5 +133,16 @@ export async function runCli(argv: string[] = process.argv): Promise { }); const processedArgv = preprocessArgv(argv); + + // Run before_session hook once at startup, before any command executes. + // Uses cwd as the search root for .agentv/config.yaml. + const cwd = process.cwd(); + const repoRoot = await findRepoRoot(cwd); + const sessionConfig = await loadConfig(path.join(cwd, '_'), repoRoot); + const beforeSessionCommand = sessionConfig?.hooks?.before_session; + if (beforeSessionCommand) { + runBeforeSessionHook(beforeSessionCommand); + } + await run(binary(app), processedArgv); } diff --git a/packages/core/src/evaluation/config.ts b/packages/core/src/evaluation/config.ts index c06f45fb2..77989b338 100644 --- a/packages/core/src/evaluation/config.ts +++ b/packages/core/src/evaluation/config.ts @@ -83,13 +83,13 @@ const AgentVConfigSchema = z.object({ hooks: z .object({ /** - * Shell command to run before the eval starts. + * Shell command to run once at agentv startup, before any command executes. * stdout is parsed for env var exports (`KEY=value` or `export KEY="value"`) * and injected into process.env. Keys already set in the environment are * not overwritten — existing env always takes priority. - * stderr is forwarded to the user. Non-zero exit aborts the eval. + * stderr is forwarded to the user. Non-zero exit aborts with an error. */ - preRun: z.string().optional(), + beforeSession: z.string().optional(), }) .optional(), }); diff --git a/packages/core/src/evaluation/hooks.ts b/packages/core/src/evaluation/hooks.ts index c36463f3b..802f4ba96 100644 --- a/packages/core/src/evaluation/hooks.ts +++ b/packages/core/src/evaluation/hooks.ts @@ -1,7 +1,7 @@ /** - * Pre-run hook execution for AgentV. + * Session hook execution for AgentV. * - * Runs a shell command before an eval starts and injects exported environment + * Runs a shell command once at agentv startup and injects exported environment * variables into the current process. This lets projects fetch secrets at * runtime (e.g. from a vault) without needing a wrapper script. * @@ -10,7 +10,7 @@ * 1. The command is run via `sh -c` (or `cmd /c` on Windows). * 2. stdout is captured and parsed for env var exports. * 3. stderr is forwarded to the process stderr so the user sees output. - * 4. Non-zero exit aborts the eval with a clear error. + * 4. Non-zero exit aborts with a clear error. * 5. Parsed keys are injected into `process.env` — only for keys not already * set, so existing env always wins. * @@ -72,7 +72,7 @@ export function parseEnvOutput(stdout: string): Record { } /** - * Run the pre_run hook command and inject exported env vars into process.env. + * Run the before_session hook command and inject exported env vars into process.env. * * - Runs via shell (`sh -c` on POSIX, `cmd /c` on Windows) * - Captured stdout is parsed for env vars; stderr is forwarded to process.stderr @@ -81,12 +81,12 @@ export function parseEnvOutput(stdout: string): Record { * * @param command Shell command string to execute */ -export function runPreRunHook(command: string): void { +export function runBeforeSessionHook(command: string): void { const isWindows = process.platform === 'win32'; const shell = isWindows ? 'cmd' : 'sh'; const shellFlag = isWindows ? '/c' : '-c'; - console.log(`${ANSI_YELLOW}Running pre-run hook: ${command}${ANSI_RESET}`); + console.log(`${ANSI_YELLOW}Running before_session hook: ${command}${ANSI_RESET}`); const result = spawnSync(shell, [shellFlag, command], { encoding: 'utf8', @@ -100,11 +100,13 @@ export function runPreRunHook(command: string): void { } if (result.error) { - throw new Error(`Pre-run hook failed to start: ${result.error.message}`); + throw new Error(`before_session hook failed to start: ${result.error.message}`); } if (result.status !== 0) { - throw new Error(`Pre-run hook exited with code ${result.status ?? 'unknown'}: ${command}`); + throw new Error( + `before_session hook exited with code ${result.status ?? 'unknown'}: ${command}`, + ); } const vars = parseEnvOutput(result.stdout ?? ''); @@ -118,6 +120,6 @@ export function runPreRunHook(command: string): void { } if (injected > 0) { - console.log(`Pre-run hook injected ${injected} environment variable(s).`); + console.log(`before_session hook injected ${injected} environment variable(s).`); } } diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index c204f9360..533869771 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -44,8 +44,8 @@ export type ResultsExportConfig = { }; export type HooksConfig = { - /** Shell command to run before the eval starts. stdout is parsed for env var exports. */ - readonly pre_run?: string; + /** Shell command to run once at agentv startup. stdout is parsed for env var exports. */ + readonly before_session?: string; }; export type AgentVConfig = { @@ -633,7 +633,7 @@ export function parseResultsExportConfig( /** * Parse the `hooks` block from .agentv/config.yaml. - * Currently supports `pre_run` only. + * Currently supports `before_session` only. */ export function parseHooksConfig(raw: unknown, configPath: string): HooksConfig | undefined { if (raw === undefined || raw === null) { @@ -646,13 +646,13 @@ export function parseHooksConfig(raw: unknown, configPath: string): HooksConfig const obj = raw as Record; - const preRun = obj.pre_run; - if (preRun !== undefined) { - if (typeof preRun !== 'string' || preRun.trim().length === 0) { - logWarning(`Invalid hooks.pre_run in ${configPath}, expected non-empty string`); + const beforeSession = obj.before_session; + if (beforeSession !== undefined) { + if (typeof beforeSession !== 'string' || beforeSession.trim().length === 0) { + logWarning(`Invalid hooks.before_session in ${configPath}, expected non-empty string`); return undefined; } - return { pre_run: preRun.trim() }; + return { before_session: beforeSession.trim() }; } return undefined; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index da8887e8b..d77011e4c 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -129,7 +129,7 @@ export { } from './evaluation/graders/assertions.js'; export { discoverGraders } from './evaluation/registry/grader-discovery.js'; export { RunBudgetTracker } from './evaluation/run-budget-tracker.js'; -export { runPreRunHook, parseEnvOutput } from './evaluation/hooks.js'; +export { runBeforeSessionHook, parseEnvOutput } from './evaluation/hooks.js'; // Import pipeline export * from './import/index.js'; diff --git a/plugins/agentv-dev/skills/agentv-eval-writer/references/config-schema.json b/plugins/agentv-dev/skills/agentv-eval-writer/references/config-schema.json index 58417af73..f95da9c49 100644 --- a/plugins/agentv-dev/skills/agentv-eval-writer/references/config-schema.json +++ b/plugins/agentv-dev/skills/agentv-eval-writer/references/config-schema.json @@ -47,11 +47,11 @@ }, "hooks": { "type": "object", - "description": "Lifecycle hooks that run at specific points in the eval lifecycle.", + "description": "Lifecycle hooks that run at specific points during agentv execution.", "properties": { - "pre_run": { + "before_session": { "type": "string", - "description": "Shell command to run before the eval starts. stdout is parsed for env var exports (KEY=value or export KEY=\"value\") and injected into process.env. Keys already set in the environment are not overwritten. stderr is forwarded to the user. Non-zero exit aborts the eval.", + "description": "Shell command to run once at agentv startup, before any command executes. stdout is parsed for env var exports (KEY=value or export KEY=\"value\") and injected into process.env. Keys already set in the environment are not overwritten. stderr is forwarded to the user. Non-zero exit aborts with an error.", "examples": ["bun scripts/load-secrets.ts", "eval $(aws ssm get-parameters-by-path ...)"] } }, From 30ee637be5e95b95afb7cbcfb27ed36673d971c8 Mon Sep 17 00:00:00 2001 From: Christopher Date: Thu, 23 Apr 2026 13:11:54 +1000 Subject: [PATCH 7/7] style: fix import order and blank line Co-Authored-By: Claude Sonnet 4.6 --- apps/cli/src/commands/eval/run-eval.ts | 1 - apps/cli/src/index.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index f428a2791..9d3cafe80 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -994,7 +994,6 @@ export async function runEvalCommand( }); } - let options = normalizeOptions(input.rawOptions, config, yamlConfig?.execution); if (!process.env.AGENTV_EXPERIMENT) { process.env.AGENTV_EXPERIMENT = normalizeExperimentName(options.experiment); diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index d55ce5863..a100a1b8f 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -1,6 +1,6 @@ import path from 'node:path'; -import { binary, run, subcommands } from 'cmd-ts'; import { loadConfig, runBeforeSessionHook } from '@agentv/core'; +import { binary, run, subcommands } from 'cmd-ts'; import { findRepoRoot } from './commands/eval/shared.js'; import packageJson from '../package.json' with { type: 'json' };