diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index b1dfa5cd4..9f054b727 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -2274,9 +2274,29 @@ export async function runEvalCommand( (sum, meta) => sum + meta.testCases.length, 0, ); + const evalTestIds = [...fileMetadata.values()].flatMap((meta) => + meta.testCases.map((testCase) => testCase.id), + ); if (transcriptProvider.lineCount !== totalTests) { throw new Error( - `Transcript has ${transcriptProvider.lineCount} entr${transcriptProvider.lineCount === 1 ? 'y' : 'ies'} but eval defines ${totalTests} test(s). Each transcript entry maps positionally to one test case.`, + `Transcript has ${transcriptProvider.lineCount} entr${transcriptProvider.lineCount === 1 ? 'y' : 'ies'} but eval defines ${totalTests} test(s). Each transcript entry must map to one test case by test_id.`, + ); + } + const transcriptTestIds = new Set(transcriptProvider.testIds); + const evalTestIdSet = new Set(evalTestIds); + const missing = evalTestIds.filter((testId) => !transcriptTestIds.has(testId)); + const extra = transcriptProvider.testIds.filter((testId) => !evalTestIdSet.has(testId)); + if (missing.length > 0 || extra.length > 0) { + throw new Error( + [ + 'Transcript test_id values must match eval test ids for replay.', + missing.length > 0 ? `Missing transcript entries: ${missing.join(', ')}` : undefined, + extra.length > 0 + ? `Transcript entries without eval tests: ${extra.join(', ')}` + : undefined, + ] + .filter((line): line is string => line !== undefined) + .join(' '), ); } diff --git a/apps/cli/src/commands/import/claude.ts b/apps/cli/src/commands/import/claude.ts index aca1a6a33..8df412509 100644 --- a/apps/cli/src/commands/import/claude.ts +++ b/apps/cli/src/commands/import/claude.ts @@ -29,6 +29,16 @@ export const importClaudeCommand = command({ description: 'Output file path (default: .agentv/transcripts/claude-.jsonl)', }), + testId: option({ + type: optional(string), + long: 'test-id', + description: 'Set the transcript test_id to match an eval test id', + }), + target: option({ + type: optional(string), + long: 'target', + description: 'Set the transcript target/source_target value (default: claude)', + }), projectsDir: option({ type: optional(string), long: 'projects-dir', @@ -39,7 +49,7 @@ export const importClaudeCommand = command({ description: 'List available sessions instead of importing', }), }, - handler: async ({ sessionId, projectPath, output, projectsDir, list }) => { + handler: async ({ sessionId, projectPath, output, testId, target, projectsDir, list }) => { if (list) { const sessions = await discoverClaudeSessions({ projectPath, @@ -95,7 +105,7 @@ export const importClaudeCommand = command({ await mkdir(path.dirname(outputPath), { recursive: true }); // Write transcript as JSONL (one message per line, grouped by test_id) - const jsonLines = toTranscriptJsonLines(transcript); + const jsonLines = toTranscriptJsonLines(transcript, transcriptWriteOptions(testId, target)); await writeFile( outputPath, `${jsonLines.map((line) => JSON.stringify(line)).join('\n')}\n`, @@ -131,6 +141,16 @@ function formatAge(date: Date): string { return `${diffDays}d ago`; } +function transcriptWriteOptions( + testId: string | undefined, + target: string | undefined, +): { testId?: string; target?: string } { + return { + ...(testId ? { testId } : {}), + ...(target ? { target } : {}), + }; +} + function formatDurationMs(ms: number): string { if (ms < 1000) return `${ms}ms`; const seconds = Math.floor(ms / 1000); diff --git a/apps/cli/src/commands/import/codex.ts b/apps/cli/src/commands/import/codex.ts index 13305a02c..259a15e95 100644 --- a/apps/cli/src/commands/import/codex.ts +++ b/apps/cli/src/commands/import/codex.ts @@ -28,6 +28,16 @@ export const importCodexCommand = command({ short: 'o', description: 'Output file path (default: .agentv/transcripts/codex-.jsonl)', }), + testId: option({ + type: optional(string), + long: 'test-id', + description: 'Set the transcript test_id to match an eval test id', + }), + target: option({ + type: optional(string), + long: 'target', + description: 'Set the transcript target/source_target value (default: codex)', + }), sessionsDir: option({ type: optional(string), long: 'sessions-dir', @@ -38,7 +48,7 @@ export const importCodexCommand = command({ description: 'List available sessions instead of importing', }), }, - handler: async ({ sessionId, date, output, sessionsDir, list }) => { + handler: async ({ sessionId, date, output, testId, target, sessionsDir, list }) => { if (list) { const sessions = await discoverCodexSessions({ date, @@ -92,7 +102,7 @@ export const importCodexCommand = command({ await mkdir(path.dirname(outputPath), { recursive: true }); // Write transcript as JSONL (one message per line, grouped by test_id) - const jsonLines = toTranscriptJsonLines(transcript); + const jsonLines = toTranscriptJsonLines(transcript, transcriptWriteOptions(testId, target)); await writeFile( outputPath, `${jsonLines.map((line) => JSON.stringify(line)).join('\n')}\n`, @@ -123,6 +133,16 @@ function formatAge(date: Date): string { return `${diffDays}d ago`; } +function transcriptWriteOptions( + testId: string | undefined, + target: string | undefined, +): { testId?: string; target?: string } { + return { + ...(testId ? { testId } : {}), + ...(target ? { target } : {}), + }; +} + function formatDurationMs(ms: number): string { if (ms < 1000) return `${ms}ms`; const seconds = Math.floor(ms / 1000); diff --git a/apps/cli/src/commands/import/copilot.ts b/apps/cli/src/commands/import/copilot.ts index 0915693f7..64fa27fa3 100644 --- a/apps/cli/src/commands/import/copilot.ts +++ b/apps/cli/src/commands/import/copilot.ts @@ -19,6 +19,16 @@ export const importCopilotCommand = command({ description: 'Output file path (default: .agentv/transcripts/copilot-.jsonl)', }), + testId: option({ + type: optional(string), + long: 'test-id', + description: 'Set the transcript test_id to match an eval test id', + }), + target: option({ + type: optional(string), + long: 'target', + description: 'Set the transcript target/source_target value (default: copilot)', + }), sessionStateDir: option({ type: optional(string), long: 'session-state-dir', @@ -29,7 +39,7 @@ export const importCopilotCommand = command({ description: 'List available sessions instead of importing', }), }, - handler: async ({ sessionId, output, sessionStateDir, list }) => { + handler: async ({ sessionId, output, testId, target, sessionStateDir, list }) => { if (list) { const sessions = await discoverCopilotSessions({ sessionStateDir, @@ -100,7 +110,7 @@ export const importCopilotCommand = command({ await mkdir(path.dirname(outputPath), { recursive: true }); // Write transcript as JSONL (one message per line, grouped by test_id) - const jsonLines = toTranscriptJsonLines(transcript); + const jsonLines = toTranscriptJsonLines(transcript, transcriptWriteOptions(testId, target)); await writeFile( outputPath, `${jsonLines.map((line) => JSON.stringify(line)).join('\n')}\n`, @@ -136,6 +146,16 @@ function formatAge(date: Date): string { return `${diffDays}d ago`; } +function transcriptWriteOptions( + testId: string | undefined, + target: string | undefined, +): { testId?: string; target?: string } { + return { + ...(testId ? { testId } : {}), + ...(target ? { target } : {}), + }; +} + function formatDurationMs(ms: number): string { if (ms < 1000) return `${ms}ms`; const seconds = Math.floor(ms / 1000); diff --git a/apps/web/src/content/docs/docs/next/guides/skill-improvement-workflow.mdx b/apps/web/src/content/docs/docs/next/guides/skill-improvement-workflow.mdx index d767cc410..4e4e27049 100644 --- a/apps/web/src/content/docs/docs/next/guides/skill-improvement-workflow.mdx +++ b/apps/web/src/content/docs/docs/next/guides/skill-improvement-workflow.mdx @@ -132,7 +132,7 @@ agentv import claude --list agentv import claude --session-id # Run deterministic graders against the imported transcript -agentv eval EVAL.yaml --target copilot-log +agentv eval EVAL.yaml --transcript .agentv/transcripts/claude-.jsonl ``` Offline grading is useful when you want to evaluate skills with agents that don't have a direct API integration — import the session transcript and run deterministic graders. diff --git a/apps/web/src/content/docs/docs/next/integrations/agent-skills-evals.mdx b/apps/web/src/content/docs/docs/next/integrations/agent-skills-evals.mdx index 61de784f0..c566fbbb7 100644 --- a/apps/web/src/content/docs/docs/next/integrations/agent-skills-evals.mdx +++ b/apps/web/src/content/docs/docs/next/integrations/agent-skills-evals.mdx @@ -122,7 +122,7 @@ Grade existing agent sessions offline by importing transcripts and running the a agentv import claude --list agentv import claude --session-id -agentv eval evals.json --target copilot-log +agentv eval evals.json --transcript .agentv/transcripts/claude-.jsonl ``` If another tool owns the original `evals.json`, keep that file as the source and run it through the read adapter. Convert only when you need to edit the AgentV-native form. diff --git a/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx b/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx index 4b2822ed8..8cfdb45c1 100644 --- a/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx +++ b/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx @@ -110,7 +110,6 @@ targets: | `claude-cli` | Claude CLI subprocess. | Default Claude path; captures structured stream output when available. | | `claude-sdk` | Claude Agent SDK in an AgentV child runner. | Explicit SDK path; useful when SDK-native events matter more than matching a local CLI invocation. | | `copilot-cli` | Copilot CLI subprocess/protocol path. | Active Copilot eval run through the installed process. | -| `copilot-log` | Passive Copilot session-log reader. | Zero-cost transcript grading for existing sessions; it does not run a new agent. | | `copilot-sdk` | Copilot SDK in an AgentV child runner. | Explicit SDK path with child-process isolation. | Every coding-agent provider returns a structured target execution envelope. @@ -271,15 +270,20 @@ targets: api_format: responses ``` -Read an existing Copilot session log without running a new agent: +Replay an existing Copilot session without running a new agent by importing the +native log into AgentV transcript rows, then using the generic replay target: + +```bash +agentv import copilot --session-id -o .agentv/transcripts/copilot-case.jsonl +``` ```yaml targets: - - id: copilot-session-log - provider: copilot-log - runtime: host + - id: copilot-session-replay + provider: replay config: - discover: latest + transcripts: .agentv/transcripts/copilot-case.jsonl + source_target: copilot-cli ``` Use `copilot-sdk` only when you intentionally want the SDK path: @@ -295,8 +299,9 @@ targets: Copilot config fields include `command`, `model`, `cwd`, `timeout_seconds`, `subprovider`, `base_url`, `api_key`, `bearer_token`, `api_version`, -`api_format`, `log_dir`, `stream_log`, `system_prompt`, and session-log fields -such as `discover`, `session_id`, and `session_dir` for `copilot-log`. +`api_format`, `log_dir`, `stream_log`, and `system_prompt`. Copilot +`events.jsonl` parsing is available through `agentv import copilot`; graders +and Dashboard views consume normalized AgentV transcript/replay artifacts. ## File inputs diff --git a/apps/web/src/content/docs/docs/next/targets/configuration.mdx b/apps/web/src/content/docs/docs/next/targets/configuration.mdx index 1ae98c4c7..e25328dc7 100644 --- a/apps/web/src/content/docs/docs/next/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/next/targets/configuration.mdx @@ -206,7 +206,6 @@ already-exported secrets into `.env`. | `codex-app-server` | Agent | Codex app-server subprocess | | `codex-sdk` | Agent | Codex SDK in an isolated child runner | | `copilot-cli` | Agent | Copilot CLI subprocess | -| `copilot-log` | Agent | Passive Copilot CLI session log reader | | `copilot-sdk` | Agent | Copilot SDK in an isolated child runner | | `pi-sdk` | Agent | Pi SDK in an isolated child runner | | `pi-cli` | Agent | Pi CLI subprocess | diff --git a/apps/web/src/content/docs/docs/next/tools/import.mdx b/apps/web/src/content/docs/docs/next/tools/import.mdx index 55a065fb1..6618526ff 100644 --- a/apps/web/src/content/docs/docs/next/tools/import.mdx +++ b/apps/web/src/content/docs/docs/next/tools/import.mdx @@ -108,6 +108,8 @@ The transcript providers share the same core flags: | `--session-id ` | Import a specific session by UUID | | `--list` | List available sessions instead of importing | | `--output, -o ` | Custom output file path | +| `--test-id ` | Set the transcript `test_id` to match an eval test id | +| `--target ` | Set the transcript `target` / replay `source_target` value | Provider-specific flags: @@ -189,19 +191,36 @@ Token usage is aggregated from the final cumulative value per LLM request. Durat ## Workflow -Import a session, then run graders against it: +Import a session, then run graders against it. The transcript `test_id` values +must match the eval test ids so replay cannot silently grade the wrong +trajectory. ```bash # 1. List sessions and pick one agentv import claude --list # 2. Import a session by ID -agentv import claude --session-id 4c4f9e4e-e6f1-490b-a1b1-9aef543ebf22 +agentv import claude --session-id 4c4f9e4e-e6f1-490b-a1b1-9aef543ebf22 --test-id my-case # 3. Run graders against the imported transcript agentv eval evals/my-eval.yaml --transcript .agentv/transcripts/claude-4c4f9e4e.jsonl ``` +You can also use the same normalized transcript rows through a replay target: + +```yaml +targets: + - id: recorded-agent + provider: replay + transcripts: .agentv/transcripts/claude-4c4f9e4e.jsonl + source_target: claude-cli +``` + +Replay targets match by `test_id` and `source_target`, return the recorded +AgentV `Message[]` trajectory, and run graders fresh. Raw provider logs remain +provenance/debug input for importers; graders and Dashboard views consume the +normalized transcript/replay artifacts. + See `examples/features/import-claude/` for a complete working example. ## HuggingFace Datasets (SWE-bench) diff --git a/apps/web/src/content/docs/docs/v4.42.4/guides/skill-improvement-workflow.mdx b/apps/web/src/content/docs/docs/v4.42.4/guides/skill-improvement-workflow.mdx index ec96c9ac9..7c14ebeb6 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/guides/skill-improvement-workflow.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/guides/skill-improvement-workflow.mdx @@ -130,7 +130,7 @@ agentv import claude --list agentv import claude --session-id # Run deterministic graders against the imported transcript -agentv eval evals.json --target copilot-log +agentv eval evals.json --transcript .agentv/transcripts/claude-.jsonl ``` Offline grading is useful when you want to evaluate skills with agents that don't have a direct API integration — import the session transcript and run deterministic graders. diff --git a/apps/web/src/content/docs/docs/v4.42.4/integrations/agent-skills-evals.mdx b/apps/web/src/content/docs/docs/v4.42.4/integrations/agent-skills-evals.mdx index 588e88acf..513cc30fc 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/integrations/agent-skills-evals.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/integrations/agent-skills-evals.mdx @@ -92,7 +92,7 @@ agentv import claude --list agentv import claude --session-id # Run deterministic graders against the imported transcript -agentv eval evals.json --target copilot-log +agentv eval evals.json --transcript .agentv/transcripts/claude-.jsonl ``` If you're using the `agentv-bench` skill bundle, validate your evals before running: diff --git a/apps/web/src/content/docs/docs/v4.42.4/targets/coding-agents.mdx b/apps/web/src/content/docs/docs/v4.42.4/targets/coding-agents.mdx index 8be53e738..21a78249e 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/targets/coding-agents.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/targets/coding-agents.mdx @@ -112,7 +112,6 @@ targets: | `claude-cli` | Claude CLI subprocess. | Default Claude path; captures structured stream output when available. | | `claude-sdk` | Claude Agent SDK in an AgentV child runner. | Explicit SDK path; useful when SDK-native events matter more than matching a local CLI invocation. | | `copilot-cli` | Copilot CLI subprocess/protocol path. | Active Copilot eval run through the installed process. | -| `copilot-log` | Passive Copilot session-log reader. | Zero-cost transcript grading for existing sessions; it does not run a new agent. | | `copilot-sdk` | Copilot SDK in an AgentV child runner. | Explicit SDK path with child-process isolation. | Every coding-agent provider returns a structured target execution envelope. @@ -273,15 +272,20 @@ targets: api_format: responses ``` -Read an existing Copilot session log without running a new agent: +Replay an existing Copilot session without running a new agent by importing the +native log into AgentV transcript rows, then using the generic replay target: + +```bash +agentv import copilot --session-id -o .agentv/transcripts/copilot-case.jsonl +``` ```yaml targets: - - id: copilot-session-log - provider: copilot-log - runtime: host + - id: copilot-session-replay + provider: replay config: - discover: latest + transcripts: .agentv/transcripts/copilot-case.jsonl + source_target: copilot-cli ``` Use `copilot-sdk` only when you intentionally want the SDK path: @@ -297,8 +301,9 @@ targets: Copilot config fields include `command`, `model`, `cwd`, `timeout_seconds`, `subprovider`, `base_url`, `api_key`, `bearer_token`, `api_version`, -`api_format`, `log_dir`, `stream_log`, `system_prompt`, and session-log fields -such as `discover`, `session_id`, and `session_dir` for `copilot-log`. +`api_format`, `log_dir`, `stream_log`, and `system_prompt`. Copilot +`events.jsonl` parsing is available through `agentv import copilot`; graders +and Dashboard views consume normalized AgentV transcript/replay artifacts. ## File inputs diff --git a/apps/web/src/content/docs/docs/v4.42.4/targets/configuration.mdx b/apps/web/src/content/docs/docs/v4.42.4/targets/configuration.mdx index 5fe634545..300cabf51 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/targets/configuration.mdx @@ -208,7 +208,6 @@ already-exported secrets into `.env`. | `codex-app-server` | Agent | Codex app-server subprocess | | `codex-sdk` | Agent | Codex SDK in an isolated child runner | | `copilot-cli` | Agent | Copilot CLI subprocess | -| `copilot-log` | Agent | Passive Copilot CLI session log reader | | `copilot-sdk` | Agent | Copilot SDK in an isolated child runner | | `pi-sdk` | Agent | Pi SDK in an isolated child runner | | `pi-cli` | Agent | Pi CLI subprocess | diff --git a/apps/web/src/content/docs/docs/v4.42.4/tools/import.mdx b/apps/web/src/content/docs/docs/v4.42.4/tools/import.mdx index d9f4d8890..165dfe6b0 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/tools/import.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/tools/import.mdx @@ -144,6 +144,8 @@ All three providers share the same core flags: | `--session-id ` | Import a specific session by UUID | | `--list` | List available sessions instead of importing | | `--output, -o ` | Custom output file path | +| `--test-id ` | Set the transcript `test_id` to match an eval test id | +| `--target ` | Set the transcript `target` / replay `source_target` value | Provider-specific flags: @@ -196,19 +198,36 @@ Token usage is aggregated from the final cumulative value per LLM request. Durat ## Workflow -Import a session, then run graders against it: +Import a session, then run graders against it. The transcript `test_id` values +must match the eval test ids so replay cannot silently grade the wrong +trajectory. ```bash # 1. List sessions and pick one agentv import claude --list # 2. Import a session by ID -agentv import claude --session-id 4c4f9e4e-e6f1-490b-a1b1-9aef543ebf22 +agentv import claude --session-id 4c4f9e4e-e6f1-490b-a1b1-9aef543ebf22 --test-id my-case # 3. Run graders against the imported transcript agentv eval evals/my-eval.yaml --transcript .agentv/transcripts/claude-4c4f9e4e.jsonl ``` +You can also use the same normalized transcript rows through a replay target: + +```yaml +targets: + - id: recorded-agent + provider: replay + transcripts: .agentv/transcripts/claude-4c4f9e4e.jsonl + source_target: claude-cli +``` + +Replay targets match by `test_id` and `source_target`, return the recorded +AgentV `Message[]` trajectory, and run graders fresh. Raw provider logs remain +provenance/debug input for importers; graders and Dashboard views consume the +normalized transcript/replay artifacts. + See `examples/features/import-claude/` for a complete working example. ## HuggingFace Datasets (SWE-bench) diff --git a/docs/plans/2026-07-01-001-feat-promptfoo-compatible-extensions-plan.md b/docs/plans/2026-07-01-001-feat-promptfoo-compatible-extensions-plan.md index 2f341e85b..d6000cbbf 100644 --- a/docs/plans/2026-07-01-001-feat-promptfoo-compatible-extensions-plan.md +++ b/docs/plans/2026-07-01-001-feat-promptfoo-compatible-extensions-plan.md @@ -206,7 +206,7 @@ The same shape can later be mirrored in AgentV examples with non-sensitive fixtu - **Dependencies:** U3, U4 - **Files:** `packages/extensions/skills/src/index.ts`, `packages/extensions/skills/src/types.ts`, `packages/extensions/skills/test/skills-extension.test.ts`, `packages/core/src/evaluation/loaders/agent-skills-parser.ts`, `packages/core/src/evaluation/providers/claude-cli.ts`, `packages/core/src/evaluation/providers/copilot-sdk.ts`, `packages/core/src/evaluation/providers/codex-cli.ts`, `packages/core/src/evaluation/providers/pi-cli.ts`, `packages/core/test/evaluation/loaders/agent-skills-parser.test.ts`, `packages/core/test/evaluation/providers/copilot-sdk.test.ts` - **Approach:** Let the skills extension copy or generate skill directories into the prepared runtime directory and return normalized `skill_paths` provider context. Providers that support explicit skill paths should consume those paths from provider request context. Existing `metadata.agent_skills_files` handling can remain as import compatibility but should not be the preferred authoring path. -- **Patterns to follow:** `packages/core/src/evaluation/loaders/agent-skills-parser.ts` for Agent Skills import, `packages/core/src/evaluation/providers/copilot-sdk.ts` for auto-discovered skill directories, and `examples/features/copilot-log-eval/` for skill-trigger evidence. +- **Patterns to follow:** `packages/core/src/evaluation/loaders/agent-skills-parser.ts` for Agent Skills import, `packages/core/src/evaluation/providers/copilot-sdk.ts` for auto-discovered skill directories, and `examples/features/copilot-transcript-replay/` for skill-trigger evidence from normalized replay transcripts. - **Test scenarios:** - A skills extension stages a `SKILL.md` directory into a prepared workspace and exposes the staged path to a provider request. - Multiple skills can be staged without overwriting each other. diff --git a/docs/plans/2026-07-03-coding-agent-target-runtime-contract.md b/docs/plans/2026-07-03-coding-agent-target-runtime-contract.md index c61d9c0e5..fa18a2951 100644 --- a/docs/plans/2026-07-03-coding-agent-target-runtime-contract.md +++ b/docs/plans/2026-07-03-coding-agent-target-runtime-contract.md @@ -287,9 +287,13 @@ an in-process adapter risk unless wrapped by a child runner. Keep provider names explicit by control boundary: - `copilot-cli`: subprocess/protocol CLI path. -- `copilot-log`: passive transcript/log replay path. - `copilot-sdk`: explicit SDK path, internally isolated if retained. +Update, 2026-07-05: the authored `copilot-log` target provider was removed. +Copilot `events.jsonl` remains an import adapter source through +`agentv import copilot`; offline grading uses normalized AgentV transcript rows +with `provider: replay` and `transcripts`, not a provider-specific log target. + ## Implementation Units ### U1. Target Schema And Docs (`av-y7eq.1`) diff --git a/examples/features/README.md b/examples/features/README.md index 8921a9e48..357b6061f 100644 --- a/examples/features/README.md +++ b/examples/features/README.md @@ -60,7 +60,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the ### Evaluate without re-running the agent (offline) | Example | Description | |---------|-------------| -| [copilot-log-eval](copilot-log-eval/) | Replay Copilot CLI session transcripts from disk — no LLM API key needed | +| [copilot-transcript-replay](copilot-transcript-replay/) | Replay normalized Copilot CLI transcript rows — no LLM API key needed | | [trace-analysis](trace-analysis/) | Inspect eval results with `agentv trace` — summaries, trees, latency percentiles | | [agent-skills-evals](agent-skills-evals/) | Evaluate Claude Code skills with `EVAL.yaml` or Agent Skills `evals.json` | @@ -143,7 +143,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [script-grader-with-llm-calls](script-grader-with-llm-calls/) | Custom graders | | [compare](compare/) | Benchmarking | | [assert-set](assert-set/) | LLM grading | -| [copilot-log-eval](copilot-log-eval/) | Offline evaluation | +| [copilot-transcript-replay](copilot-transcript-replay/) | Offline evaluation | | [default-graders](default-graders/) | Getting started | | [deterministic-graders](deterministic-graders/) | Deterministic assertions | | [document-extraction](document-extraction/) | Document extraction | diff --git a/examples/features/copilot-log-eval/.agentv/targets.yaml b/examples/features/copilot-log-eval/.agentv/targets.yaml deleted file mode 100644 index 1b42e88db..000000000 --- a/examples/features/copilot-log-eval/.agentv/targets.yaml +++ /dev/null @@ -1,14 +0,0 @@ -targets: - # Passive transcript reader — reads Copilot CLI session transcripts from disk. - # Zero API cost. No grader_target needed for deterministic graders. - # - # Usage: - # agentv eval evals/skill-trigger.EVAL.yaml --target copilot-log - # - # Session resolution (pick one): - # discover: latest — auto-discover most recent session - # session_id: "" — read a specific session by ID - # session_dir: "/path/to/dir" — read from an explicit directory - - id: copilot-log - provider: copilot-log - discover: latest diff --git a/examples/features/copilot-transcript-replay/.agentv/targets.yaml b/examples/features/copilot-transcript-replay/.agentv/targets.yaml new file mode 100644 index 000000000..f79ff89b1 --- /dev/null +++ b/examples/features/copilot-transcript-replay/.agentv/targets.yaml @@ -0,0 +1,8 @@ +targets: + # Recorded Copilot trajectory replay. Import Copilot CLI events with + # `agentv import copilot`, or use the checked-in normalized transcript + # fixture below. Graders consume AgentV transcript rows, not raw events.jsonl. + - id: copilot-transcript-replay + provider: replay + transcripts: ../fixtures/copilot-transcript.jsonl + source_target: copilot-cli diff --git a/examples/features/copilot-log-eval/README.md b/examples/features/copilot-transcript-replay/README.md similarity index 56% rename from examples/features/copilot-log-eval/README.md rename to examples/features/copilot-transcript-replay/README.md index f95e87355..e6b0d0e3c 100644 --- a/examples/features/copilot-log-eval/README.md +++ b/examples/features/copilot-transcript-replay/README.md @@ -1,7 +1,9 @@ -# Copilot Log Evaluation Example +# Copilot Transcript Replay Example -Demonstrates the `copilot-log` provider reading Copilot CLI session transcripts -from disk with deterministic graders. **No LLM API key needed.** +Demonstrates provider-agnostic recorded trajectory replay for a Copilot CLI +session. Copilot `events.jsonl` data is normalized into AgentV transcript JSONL, +then `provider: replay` runs deterministic graders without invoking Copilot +again. **No LLM API key needed for replay.** Graders used: - `skill-trigger` — checks whether a specific skill was invoked @@ -25,15 +27,28 @@ copilot --model gpt-5-mini > Analyze this CSV file and tell me the top 5 months by revenue ``` -### 2. Run the eval +Import the session into a normalized AgentV transcript: ```bash -agentv eval evals/skill-trigger.EVAL.yaml --target copilot-log +agentv import copilot \ + --session-id \ + --test-id should-not-trigger-csv-analyzer \ + --target copilot-cli \ + -o fixtures/copilot-transcript.jsonl ``` -The `before_all` hook runs `allagents workspace init` to sync the agentv-dev -plugin skills into the workspace. The `copilot-log` provider then auto-discovers -the latest session from `~/.copilot/session-state/` and runs all graders. +The checked-in fixture already uses this normalized shape for deterministic +example runs. + +### 2. Run the replay eval + +```bash +agentv eval evals/skill-trigger.EVAL.yaml --target copilot-transcript-replay +``` + +The `before_all` hook syncs the agentv-dev plugin skills into the workspace. +The replay target matches the eval case by `test_id`, reads the normalized +transcript rows from `fixtures/copilot-transcript.jsonl`, and runs all graders. ## How it works @@ -41,8 +56,10 @@ the latest session from `~/.copilot/session-state/` and runs all graders. allagents workspace init (setup hook) ↓ syncs agentv-dev plugin skills from marketplace ~/.copilot/session-state/{uuid}/events.jsonl - ↓ copilot-log provider (reads from disk) -Message[] with tool calls + ↓ agentv import copilot +AgentV transcript JSONL (agentv.transcript.v1) + ↓ provider: replay with transcripts: fixtures/copilot-transcript.jsonl +Message[] with tool calls and raw source provenance ├─ skill-trigger grader (deterministic) → pass/fail └─ script-grader (graders/transcript-quality.ts) → pass/fail ``` diff --git a/examples/features/copilot-log-eval/evals/skill-trigger.EVAL.yaml b/examples/features/copilot-transcript-replay/evals/skill-trigger.EVAL.yaml similarity index 83% rename from examples/features/copilot-log-eval/evals/skill-trigger.EVAL.yaml rename to examples/features/copilot-transcript-replay/evals/skill-trigger.EVAL.yaml index dff013cdc..2d1841b23 100644 --- a/examples/features/copilot-log-eval/evals/skill-trigger.EVAL.yaml +++ b/examples/features/copilot-transcript-replay/evals/skill-trigger.EVAL.yaml @@ -1,8 +1,8 @@ tags: - agent -target: copilot-log +target: copilot-transcript-replay extensions: - - file://../scripts/copilot-log-workspace.mjs:beforeAll + - file://../scripts/copilot-replay-workspace.mjs:beforeAll workspace: scope: suite template: ../workspace/ diff --git a/examples/features/copilot-transcript-replay/fixtures/copilot-transcript.jsonl b/examples/features/copilot-transcript-replay/fixtures/copilot-transcript.jsonl new file mode 100644 index 000000000..f2ef4b292 --- /dev/null +++ b/examples/features/copilot-transcript-replay/fixtures/copilot-transcript.jsonl @@ -0,0 +1,4 @@ +{"schema_version":"agentv.transcript.v1","test_id":"should-not-trigger-csv-analyzer","target":"copilot-cli","message_index":0,"role":"user","content":"Analyze this CSV file and tell me the top 5 months by revenue","transcript_token_usage":{"input":900,"output":220},"transcript_duration_ms":1750,"transcript_cost_usd":null,"capture":{"content":"full","redaction_level":"none"},"source":{"kind":"imported_transcript","provider":"copilot","session_id":"copilot-fixture-session","format":"copilot_cli_events_jsonl","model":"gpt-5-mini","timestamp":"2026-07-05T00:00:00.000Z","cwd":"/workspace/agentv","metadata":{"raw_log":"events.jsonl"}}} +{"schema_version":"agentv.transcript.v1","test_id":"should-not-trigger-csv-analyzer","target":"copilot-cli","message_index":1,"role":"assistant","content":"I will inspect the CSV contents directly and compute the top revenue months.","transcript_token_usage":{"input":900,"output":220},"transcript_duration_ms":1750,"transcript_cost_usd":null,"capture":{"content":"full","redaction_level":"none"},"source":{"kind":"imported_transcript","provider":"copilot","session_id":"copilot-fixture-session","format":"copilot_cli_events_jsonl","model":"gpt-5-mini","timestamp":"2026-07-05T00:00:00.000Z","cwd":"/workspace/agentv","metadata":{"raw_log":"events.jsonl"}}} +{"schema_version":"agentv.transcript.v1","test_id":"should-not-trigger-csv-analyzer","target":"copilot-cli","message_index":2,"role":"assistant","tool_calls":[{"tool":"Read","input":{"path":"sales.csv","file_path":"sales.csv"},"id":"call-read-sales","output":"month,revenue,units_sold\nJanuary,12500,150\nFebruary,9800,120\nMarch,15200,180\nApril,11300,140\nMay,18900,220\nJune,14700,175\nJuly,16500,195\nAugust,13200,160\nSeptember,20100,240\nOctober,17800,210\nNovember,22500,265\nDecember,19400,230"}],"transcript_token_usage":{"input":900,"output":220},"transcript_duration_ms":1750,"transcript_cost_usd":null,"capture":{"content":"full","redaction_level":"none"},"source":{"kind":"imported_transcript","provider":"copilot","session_id":"copilot-fixture-session","format":"copilot_cli_events_jsonl","model":"gpt-5-mini","timestamp":"2026-07-05T00:00:00.000Z","cwd":"/workspace/agentv","metadata":{"raw_log":"events.jsonl"}}} +{"schema_version":"agentv.transcript.v1","test_id":"should-not-trigger-csv-analyzer","target":"copilot-cli","message_index":3,"role":"assistant","content":"The top revenue months are November, September, December, May, and October. I answered from the CSV data without invoking the csv-analyzer skill.","transcript_token_usage":{"input":900,"output":220},"transcript_duration_ms":1750,"transcript_cost_usd":null,"capture":{"content":"full","redaction_level":"none"},"source":{"kind":"imported_transcript","provider":"copilot","session_id":"copilot-fixture-session","format":"copilot_cli_events_jsonl","model":"gpt-5-mini","timestamp":"2026-07-05T00:00:00.000Z","cwd":"/workspace/agentv","metadata":{"raw_log":"events.jsonl"}}} diff --git a/examples/features/copilot-log-eval/graders/transcript-quality.ts b/examples/features/copilot-transcript-replay/graders/transcript-quality.ts similarity index 94% rename from examples/features/copilot-log-eval/graders/transcript-quality.ts rename to examples/features/copilot-transcript-replay/graders/transcript-quality.ts index 9d6a49d9d..3621e0c34 100644 --- a/examples/features/copilot-log-eval/graders/transcript-quality.ts +++ b/examples/features/copilot-transcript-replay/graders/transcript-quality.ts @@ -2,12 +2,12 @@ /** * Transcript Quality Grader * - * Validates that the copilot-log provider produced a meaningful transcript: + * Validates that replayed Copilot transcript rows produced a meaningful transcript: * 1. At least one assistant message exists * 2. At least one tool call was recorded * 3. The assistant response addresses the user's question (mentions CSV-relevant terms) * - * Uses the full Message[] from the copilot-log provider, including toolCalls arrays. + * Uses the full replayed Message[] output, including toolCalls arrays. * * Usage in eval YAML: * assertions: diff --git a/examples/features/copilot-log-eval/scripts/copilot-log-workspace.mjs b/examples/features/copilot-transcript-replay/scripts/copilot-replay-workspace.mjs similarity index 94% rename from examples/features/copilot-log-eval/scripts/copilot-log-workspace.mjs rename to examples/features/copilot-transcript-replay/scripts/copilot-replay-workspace.mjs index e1cbf745d..ed1c12467 100644 --- a/examples/features/copilot-log-eval/scripts/copilot-log-workspace.mjs +++ b/examples/features/copilot-transcript-replay/scripts/copilot-replay-workspace.mjs @@ -12,7 +12,7 @@ const REQUIRED_FILES = ['.github/skills/agentv-bench/SKILL.md']; export function beforeAll(context) { const workspacePath = context.workspace_path; if (!workspacePath) { - throw new Error('workspace_path not provided to copilot-log setup extension'); + throw new Error('workspace_path not provided to copilot replay setup extension'); } rmSync(join(workspacePath, '.allagents'), { recursive: true, force: true }); diff --git a/examples/features/copilot-log-eval/workspace/.allagents/workspace.yaml b/examples/features/copilot-transcript-replay/workspace/.allagents/workspace.yaml similarity index 100% rename from examples/features/copilot-log-eval/workspace/.allagents/workspace.yaml rename to examples/features/copilot-transcript-replay/workspace/.allagents/workspace.yaml diff --git a/examples/features/copilot-log-eval/workspace/.copilot/skills/csv-analyzer/SKILL.md b/examples/features/copilot-transcript-replay/workspace/.copilot/skills/csv-analyzer/SKILL.md similarity index 100% rename from examples/features/copilot-log-eval/workspace/.copilot/skills/csv-analyzer/SKILL.md rename to examples/features/copilot-transcript-replay/workspace/.copilot/skills/csv-analyzer/SKILL.md diff --git a/examples/features/copilot-log-eval/workspace/AGENTS.md b/examples/features/copilot-transcript-replay/workspace/AGENTS.md similarity index 89% rename from examples/features/copilot-log-eval/workspace/AGENTS.md rename to examples/features/copilot-transcript-replay/workspace/AGENTS.md index 54c49296d..0cb45f184 100644 --- a/examples/features/copilot-log-eval/workspace/AGENTS.md +++ b/examples/features/copilot-transcript-replay/workspace/AGENTS.md @@ -1,4 +1,4 @@ -# Workspace for copilot-log eval testing +# Workspace for Copilot transcript replay eval testing This workspace contains skills for skill-trigger evaluation. diff --git a/examples/features/copilot-log-eval/workspace/sales.csv b/examples/features/copilot-transcript-replay/workspace/sales.csv similarity index 100% rename from examples/features/copilot-log-eval/workspace/sales.csv rename to examples/features/copilot-transcript-replay/workspace/sales.csv diff --git a/examples/features/import-claude/README.md b/examples/features/import-claude/README.md index aad1c4d4f..67240aa69 100644 --- a/examples/features/import-claude/README.md +++ b/examples/features/import-claude/README.md @@ -23,13 +23,13 @@ claude -p "List all TypeScript files in this project" agentv import claude --list # Import by session ID -agentv import claude --session-id -o transcripts/session.jsonl +agentv import claude --session-id --test-id transcript-quality -o transcripts/session.jsonl ``` ### 3. Run the eval ```bash -agentv eval evals/transcript-check.EVAL.yaml +agentv eval evals/transcript-check.EVAL.yaml --transcript transcripts/session.jsonl ``` ## How it works @@ -38,6 +38,7 @@ agentv eval evals/transcript-check.EVAL.yaml ~/.claude/projects//.jsonl ↓ agentv import claude (reads from disk, converts to Message[]) .agentv/transcripts/claude-.jsonl + ↓ agentv eval --transcript ↓ script-grader (deterministic) pass/fail ``` diff --git a/examples/showcase/trace-evaluation/.agentv/targets.yaml b/examples/showcase/trace-evaluation/.agentv/targets.yaml index bf4a069ea..4c2a1511a 100644 --- a/examples/showcase/trace-evaluation/.agentv/targets.yaml +++ b/examples/showcase/trace-evaluation/.agentv/targets.yaml @@ -1,6 +1,6 @@ targets: - id: live_coding_agent - provider: codex + provider: codex-cli model: gpt-5 timeout_seconds: 300 @@ -9,3 +9,8 @@ targets: fixtures: ../fixtures/replay-target-output.jsonl suite: trace-evaluation-showcase source_target: live_coding_agent + + - id: replay_imported_codex_transcript + provider: replay + transcripts: ../fixtures/imported-codex-transcript.jsonl + source_target: codex diff --git a/examples/showcase/trace-evaluation/README.md b/examples/showcase/trace-evaluation/README.md index dc436f2e6..05e3065f6 100644 --- a/examples/showcase/trace-evaluation/README.md +++ b/examples/showcase/trace-evaluation/README.md @@ -42,8 +42,8 @@ The replay target looks up records by `suite`, `eval_path` when present, `test_i records fail before grading. Replay can also read `agentv.trace.v1` artifacts by using -`execution_traces` instead of `fixtures` on the replay target. Configure exactly -one source field: +`execution_traces` or normalized AgentV transcript JSONL by using `transcripts` +instead of `fixtures` on the replay target. Configure exactly one source field: ```yaml targets: @@ -58,6 +58,10 @@ Execution trace replay requires the matched artifact to contain full captured as output. Metadata-only trace sidecars fail before grading rather than replaying an empty answer. +Transcript replay uses `agentv.transcript.v1` rows from `agentv import` and +matches by `test_id` plus `source_target`. A missing or mismatched `test_id` +fails before grading so a grader does not silently score the wrong trajectory. + ## Proof Run ```bash @@ -84,13 +88,16 @@ bun apps/cli/src/cli.ts eval \ ## Transcript Import Fixture -The imported fixture was produced through the existing Codex import command: +The imported fixture was produced through the Codex import command with +`--test-id` set to the eval case id. The row `source.session_id` still +preserves the raw session provenance: ```bash bun apps/cli/src/cli.ts import codex \ --sessions-dir examples/showcase/trace-evaluation/fixtures/raw/codex-sessions \ --date 2026-06-06 \ --session-id 00000000-0000-4000-8000-000000000001 \ + --test-id imported-codex-config-fix \ --output examples/showcase/trace-evaluation/fixtures/imported-codex-transcript.jsonl ``` diff --git a/examples/showcase/trace-evaluation/evals/transcript-import.eval.yaml b/examples/showcase/trace-evaluation/evals/transcript-import.eval.yaml index de33cc26f..8f6fb4e56 100644 --- a/examples/showcase/trace-evaluation/evals/transcript-import.eval.yaml +++ b/examples/showcase/trace-evaluation/evals/transcript-import.eval.yaml @@ -30,9 +30,6 @@ tests: - run - ../graders/replay-proof.ts require_metrics: false - - |- - The imported coding-agent transcript should show a repository inspection, - a targeted config edit, and a concise final explanation. vars: input: - role: user diff --git a/examples/showcase/trace-evaluation/fixtures/imported-codex-transcript.jsonl b/examples/showcase/trace-evaluation/fixtures/imported-codex-transcript.jsonl index bb20cc4c3..b7ed382d9 100644 --- a/examples/showcase/trace-evaluation/fixtures/imported-codex-transcript.jsonl +++ b/examples/showcase/trace-evaluation/fixtures/imported-codex-transcript.jsonl @@ -1,6 +1,6 @@ -{"test_id":"00000000-0000-4000-8000-000000000001","target":"codex","message_index":0,"role":"user","content":"Inspect the project and change the default request timeout to 5000ms.","transcript_duration_ms":1200,"transcript_cost_usd":null,"source":{"provider":"codex","session_id":"00000000-0000-4000-8000-000000000001","model":"gpt-5","timestamp":"2026-06-06T12:00:00.000Z","cwd":"/workspace/agentv","version":"0.0.0-fixture"}} -{"test_id":"00000000-0000-4000-8000-000000000001","target":"codex","message_index":1,"role":"assistant","content":"I will inspect the project metadata and locate the timeout setting before editing.","transcript_duration_ms":1200,"transcript_cost_usd":null,"source":{"provider":"codex","session_id":"00000000-0000-4000-8000-000000000001","model":"gpt-5","timestamp":"2026-06-06T12:00:00.000Z","cwd":"/workspace/agentv","version":"0.0.0-fixture"}} -{"test_id":"00000000-0000-4000-8000-000000000001","target":"codex","message_index":2,"role":"assistant","tool_calls":[{"tool":"Read","input":{"path":"package.json"},"id":"call-read-package","output":{"name":"agentv-example-service","scripts":["test","lint"]}}],"transcript_duration_ms":1200,"transcript_cost_usd":null,"source":{"provider":"codex","session_id":"00000000-0000-4000-8000-000000000001","model":"gpt-5","timestamp":"2026-06-06T12:00:00.000Z","cwd":"/workspace/agentv","version":"0.0.0-fixture"}} -{"test_id":"00000000-0000-4000-8000-000000000001","target":"codex","message_index":3,"role":"assistant","tool_calls":[{"tool":"Grep","input":{"pattern":"timeout","path":"src"},"id":"call-grep-timeout","output":{"matches":["src/config.ts:12: timeoutMs: 0"]}}],"transcript_duration_ms":1200,"transcript_cost_usd":null,"source":{"provider":"codex","session_id":"00000000-0000-4000-8000-000000000001","model":"gpt-5","timestamp":"2026-06-06T12:00:00.000Z","cwd":"/workspace/agentv","version":"0.0.0-fixture"}} -{"test_id":"00000000-0000-4000-8000-000000000001","target":"codex","message_index":4,"role":"assistant","tool_calls":[{"tool":"Edit","input":{"path":"src/config.ts","old_string":"timeoutMs: 0","new_string":"timeoutMs: 5000"},"id":"call-edit-timeout","output":{"changed":true,"diff":"- timeoutMs: 0\n+ timeoutMs: 5000"}}],"transcript_duration_ms":1200,"transcript_cost_usd":null,"source":{"provider":"codex","session_id":"00000000-0000-4000-8000-000000000001","model":"gpt-5","timestamp":"2026-06-06T12:00:00.000Z","cwd":"/workspace/agentv","version":"0.0.0-fixture"}} -{"test_id":"00000000-0000-4000-8000-000000000001","target":"codex","message_index":5,"role":"assistant","content":"Updated src/config.ts so the default request timeout is 5000ms. I inspected package.json first, found the timeout setting with Grep, and only edited the config file.","transcript_duration_ms":1200,"transcript_cost_usd":null,"source":{"provider":"codex","session_id":"00000000-0000-4000-8000-000000000001","model":"gpt-5","timestamp":"2026-06-06T12:00:00.000Z","cwd":"/workspace/agentv","version":"0.0.0-fixture"}} +{"test_id":"imported-codex-config-fix","target":"codex","message_index":0,"role":"user","content":"Inspect the project and change the default request timeout to 5000ms.","transcript_duration_ms":1200,"transcript_cost_usd":null,"source":{"provider":"codex","session_id":"00000000-0000-4000-8000-000000000001","model":"gpt-5","timestamp":"2026-06-06T12:00:00.000Z","cwd":"/workspace/agentv","version":"0.0.0-fixture"}} +{"test_id":"imported-codex-config-fix","target":"codex","message_index":1,"role":"assistant","content":"I will inspect the project metadata and locate the timeout setting before editing.","transcript_duration_ms":1200,"transcript_cost_usd":null,"source":{"provider":"codex","session_id":"00000000-0000-4000-8000-000000000001","model":"gpt-5","timestamp":"2026-06-06T12:00:00.000Z","cwd":"/workspace/agentv","version":"0.0.0-fixture"}} +{"test_id":"imported-codex-config-fix","target":"codex","message_index":2,"role":"assistant","tool_calls":[{"tool":"Read","input":{"path":"package.json"},"id":"call-read-package","output":{"name":"agentv-example-service","scripts":["test","lint"]}}],"transcript_duration_ms":1200,"transcript_cost_usd":null,"source":{"provider":"codex","session_id":"00000000-0000-4000-8000-000000000001","model":"gpt-5","timestamp":"2026-06-06T12:00:00.000Z","cwd":"/workspace/agentv","version":"0.0.0-fixture"}} +{"test_id":"imported-codex-config-fix","target":"codex","message_index":3,"role":"assistant","tool_calls":[{"tool":"Grep","input":{"pattern":"timeout","path":"src"},"id":"call-grep-timeout","output":{"matches":["src/config.ts:12: timeoutMs: 0"]}}],"transcript_duration_ms":1200,"transcript_cost_usd":null,"source":{"provider":"codex","session_id":"00000000-0000-4000-8000-000000000001","model":"gpt-5","timestamp":"2026-06-06T12:00:00.000Z","cwd":"/workspace/agentv","version":"0.0.0-fixture"}} +{"test_id":"imported-codex-config-fix","target":"codex","message_index":4,"role":"assistant","tool_calls":[{"tool":"Edit","input":{"path":"src/config.ts","old_string":"timeoutMs: 0","new_string":"timeoutMs: 5000"},"id":"call-edit-timeout","output":{"changed":true,"diff":"- timeoutMs: 0\n+ timeoutMs: 5000"}}],"transcript_duration_ms":1200,"transcript_cost_usd":null,"source":{"provider":"codex","session_id":"00000000-0000-4000-8000-000000000001","model":"gpt-5","timestamp":"2026-06-06T12:00:00.000Z","cwd":"/workspace/agentv","version":"0.0.0-fixture"}} +{"test_id":"imported-codex-config-fix","target":"codex","message_index":5,"role":"assistant","content":"Updated src/config.ts so the default request timeout is 5000ms. I inspected package.json first, found the timeout setting with Grep, and only edited the config file.","transcript_duration_ms":1200,"transcript_cost_usd":null,"source":{"provider":"codex","session_id":"00000000-0000-4000-8000-000000000001","model":"gpt-5","timestamp":"2026-06-06T12:00:00.000Z","cwd":"/workspace/agentv","version":"0.0.0-fixture"}} diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index ae10788d1..4f3d852fb 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -264,7 +264,7 @@ function createEvaluationRuntime(options: EvaluationRuntimeOptions): EvaluationR const resolvedGrader = resolveTargetByName(graderName); if (!resolvedGrader) { // Only use the eval target as its own grader if it can return structured JSON. - // Agent providers, transcript, cli, and copilot-log cannot grade. + // Agent providers, transcript, cli, and replay cannot grade. if (!LLM_GRADER_CAPABLE_KINDS.includes(targetContext.kind)) { return undefined; } diff --git a/packages/core/src/evaluation/providers/copilot-log-parser.ts b/packages/core/src/evaluation/providers/copilot-log-parser.ts index e1b41414d..503cc8c57 100644 --- a/packages/core/src/evaluation/providers/copilot-log-parser.ts +++ b/packages/core/src/evaluation/providers/copilot-log-parser.ts @@ -108,7 +108,7 @@ export function parseCopilotEvents(eventsJsonl: string): ParsedCopilotSession { const toolRequests = data.toolRequests as readonly Record[] | undefined; const toolCalls: ToolCall[] = (toolRequests ?? []).map((req) => - normalizeToolCall('copilot-log', { + normalizeToolCall('copilot-events', { tool: String(req.name ?? req.toolName ?? ''), input: req.arguments, id: req.toolCallId ? String(req.toolCallId) : undefined, @@ -160,7 +160,7 @@ export function parseCopilotEvents(eventsJsonl: string): ParsedCopilotSession { messages.push({ role: 'assistant', toolCalls: [ - normalizeToolCall('copilot-log', { + normalizeToolCall('copilot-events', { tool: started.toolName, input: started.input, output: data.result, diff --git a/packages/core/src/evaluation/providers/copilot-log.ts b/packages/core/src/evaluation/providers/copilot-log.ts deleted file mode 100644 index 4283ad621..000000000 --- a/packages/core/src/evaluation/providers/copilot-log.ts +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Copilot Log provider — reads Copilot CLI session transcripts from disk. - * - * Zero-cost alternative to spawning a Copilot CLI instance. Reads - * ~/.copilot/session-state/{uuid}/events.jsonl and converts to Message[]. - * - * Config options (specify ONE of these to identify the session): - * sessionDir — explicit path to a session directory - * sessionId — session UUID (combined with sessionStateDir) - * discover — 'latest' to auto-discover most recent session - * - * Optional: - * sessionStateDir — override ~/.copilot/session-state - * cwd — filter discovery by working directory - * - * The invoke() method ignores request.question since no process is spawned. - * It reads the transcript file and returns a ProviderResponse with the - * parsed Message[] in the output field. - * - * File-change tracking: - * After reading the transcript, the provider automatically scans the - * session's `files/` subdirectory for artifacts generated during the - * session (e.g. CSV / Markdown reports saved by Copilot). Any files - * found are returned as synthetic unified diffs in `fileChanges` so that - * LLM and script graders can evaluate them via `{{file_changes}}` without - * requiring the agent to echo file contents in its final answer. - */ - -import { readFile } from 'node:fs/promises'; -import { homedir } from 'node:os'; -import path from 'node:path'; -import { captureSessionArtifacts } from '../workspace/file-changes.js'; -import { parseCopilotEvents } from './copilot-log-parser.js'; -import { discoverCopilotSessions } from './copilot-session-discovery.js'; -import type { CopilotLogResolvedConfig } from './targets.js'; -import type { Provider, ProviderRequest, ProviderResponse } from './types.js'; - -export class CopilotLogProvider implements Provider { - readonly id: string; - readonly kind = 'copilot-log' as const; - readonly targetName: string; - - private readonly config: CopilotLogResolvedConfig; - - constructor(targetName: string, config: CopilotLogResolvedConfig) { - this.targetName = targetName; - this.id = `copilot-log:${targetName}`; - this.config = config; - } - - async invoke(_request: ProviderRequest): Promise { - const sessionDir = await this.resolveSessionDir(); - const eventsPath = path.join(sessionDir, 'events.jsonl'); - - let eventsContent: string; - try { - eventsContent = await readFile(eventsPath, 'utf8'); - } catch (err) { - throw new Error( - `Failed to read Copilot session transcript at ${eventsPath}: ${err instanceof Error ? err.message : String(err)}`, - ); - } - - const parsed = parseCopilotEvents(eventsContent); - - // Scan session-state `files/` directory for artifacts generated during - // the session (e.g. CSV reports). Return as synthetic diffs so graders - // can evaluate them via {{file_changes}} without special eval wiring. - const filesDir = path.join(sessionDir, 'files'); - const fileChanges = await captureSessionArtifacts(filesDir).catch(() => undefined); - - return { - output: parsed.messages, - tokenUsage: parsed.tokenUsage, - durationMs: parsed.durationMs, - startTime: parsed.meta.startedAt, - ...(fileChanges ? { fileChanges } : {}), - }; - } - - private async resolveSessionDir(): Promise { - if (this.config.sessionDir) { - return this.config.sessionDir; - } - - if (this.config.sessionId) { - const stateDir = - this.config.sessionStateDir ?? path.join(homedir(), '.copilot', 'session-state'); - return path.join(stateDir, this.config.sessionId); - } - - if (this.config.discover === 'latest') { - const sessions = await discoverCopilotSessions({ - sessionStateDir: this.config.sessionStateDir, - cwd: this.config.cwd, - limit: 1, - }); - - if (sessions.length === 0) { - throw new Error( - `No Copilot CLI sessions found${this.config.cwd ? ` for cwd=${this.config.cwd}` : ''}. ` + - `Check that sessions exist in ${this.config.sessionStateDir ?? '~/.copilot/session-state/'}`, - ); - } - - return sessions[0].sessionDir; - } - - throw new Error( - 'CopilotLogProvider requires one of: sessionDir, sessionId, or discover="latest"', - ); - } -} diff --git a/packages/core/src/evaluation/providers/index.ts b/packages/core/src/evaluation/providers/index.ts index a916d6e59..18f5ba8e9 100644 --- a/packages/core/src/evaluation/providers/index.ts +++ b/packages/core/src/evaluation/providers/index.ts @@ -3,7 +3,6 @@ import { ClaudeCliProvider } from './claude-cli.js'; import { CliProvider } from './cli.js'; import { CodexAppServerProvider, CodexCliProvider } from './codex-cli.js'; import { CopilotCliProvider } from './copilot-cli.js'; -import { CopilotLogProvider } from './copilot-log.js'; import { AnthropicProvider, AzureProvider, @@ -57,7 +56,6 @@ export type { CliResolvedConfig, CopilotCliResolvedConfig, CopilotCustomProviderConfig, - CopilotLogResolvedConfig, CopilotSdkResolvedConfig, GeminiResolvedConfig, MockResolvedConfig, @@ -213,7 +211,6 @@ export function createBuiltinProviderRegistry(): ProviderRegistry { ? unsupportedSandboxProvider(t) : new CopilotCliProvider(t.name, t.config as never), ) - .register('copilot-log', (t) => new CopilotLogProvider(t.name, t.config as never)) .register('pi-sdk', (t) => usesSandboxRuntime(t) ? unsupportedSandboxProvider(t) diff --git a/packages/core/src/evaluation/providers/normalize-tool-call.ts b/packages/core/src/evaluation/providers/normalize-tool-call.ts index 4584e64ae..60f919ad8 100644 --- a/packages/core/src/evaluation/providers/normalize-tool-call.ts +++ b/packages/core/src/evaluation/providers/normalize-tool-call.ts @@ -25,6 +25,8 @@ import type { ProviderKind } from './types.js'; import type { ToolCall } from './types.js'; +export type ToolNormalizationSource = ProviderKind | 'copilot-events'; + // --------------------------------------------------------------------------- // Canonical tool names // --------------------------------------------------------------------------- @@ -80,17 +82,17 @@ const TOOL_NAME_MAP = new Map([ ['copilot-sdk::Edit File', 'Edit'], ['copilot-sdk::runTerminalCommand', 'Bash'], - ['copilot-log::Skill', 'Skill'], - ['copilot-log::skill', 'Skill'], - ['copilot-log::Read File', 'Read'], - ['copilot-log::readFile', 'Read'], - ['copilot-log::Read', 'Read'], - ['copilot-log::readTextFile', 'Read'], - ['copilot-log::writeTextFile', 'Write'], - ['copilot-log::Write File', 'Write'], - ['copilot-log::editFile', 'Edit'], - ['copilot-log::Edit File', 'Edit'], - ['copilot-log::runTerminalCommand', 'Bash'], + ['copilot-events::Skill', 'Skill'], + ['copilot-events::skill', 'Skill'], + ['copilot-events::Read File', 'Read'], + ['copilot-events::readFile', 'Read'], + ['copilot-events::Read', 'Read'], + ['copilot-events::readTextFile', 'Read'], + ['copilot-events::writeTextFile', 'Write'], + ['copilot-events::Write File', 'Write'], + ['copilot-events::editFile', 'Edit'], + ['copilot-events::Edit File', 'Edit'], + ['copilot-events::runTerminalCommand', 'Bash'], ['vscode::Skill', 'Skill'], ['vscode::skill', 'Skill'], @@ -162,7 +164,7 @@ const CODEX_PREFIXES: readonly PrefixRule[] = [ const TOOL_PREFIX_MAP = new Map([ ['copilot-cli', COPILOT_PREFIXES], ['copilot-sdk', COPILOT_PREFIXES], - ['copilot-log', COPILOT_PREFIXES], + ['copilot-events', COPILOT_PREFIXES], ['vscode', COPILOT_PREFIXES], ['vscode-insiders', COPILOT_PREFIXES], ['codex-cli', CODEX_PREFIXES], @@ -207,7 +209,7 @@ const INPUT_NORMALIZERS = new Map([ * This is a pure function — provider kind in, canonical ToolCall out. * Unknown tool names pass through unchanged. */ -export function normalizeToolCall(providerKind: ProviderKind, tc: ToolCall): ToolCall { +export function normalizeToolCall(providerKind: ToolNormalizationSource, tc: ToolCall): ToolCall { const nativeName = tc.tool; // 1. Try exact match diff --git a/packages/core/src/evaluation/providers/replay.ts b/packages/core/src/evaluation/providers/replay.ts index 9e94cd471..37c3312a2 100644 --- a/packages/core/src/evaluation/providers/replay.ts +++ b/packages/core/src/evaluation/providers/replay.ts @@ -3,8 +3,8 @@ * * Configure it in targets.yaml with `provider: replay`, the `source_target` * whose live outputs were recorded, and exactly one replay source: `fixtures` - * JSONL or `execution_traces`. The provider does not invoke the source target; - * it only performs strict replay lookup and returns the recorded + * JSONL, `execution_traces`, or normalized `transcripts`. The provider does + * not invoke the source target; it only performs strict replay lookup and returns the recorded * ProviderResponse so graders can run fresh. */ @@ -18,6 +18,11 @@ import { readTraceEnvelopeReplayRecords, traceEnvelopeReplayRecordToProviderResponse, } from '../replay-trace-envelopes.js'; +import { + findTranscriptReplayRecord, + readTranscriptReplayRecords, + transcriptReplayRecordToProviderResponse, +} from '../replay-transcripts.js'; import type { ReplayResolvedConfig } from './targets.js'; import type { Provider, ProviderRequest, ProviderResponse } from './types.js'; @@ -48,6 +53,11 @@ export class ReplayProvider implements Provider { const record = findTraceEnvelopeReplayRecord(records, this.lookupForRequest(request)); return traceEnvelopeReplayRecordToProviderResponse(record); } + case 'transcripts': { + const records = await readTranscriptReplayRecords(source.path); + const record = findTranscriptReplayRecord(records, this.lookupForRequest(request)); + return transcriptReplayRecordToProviderResponse(record); + } } } @@ -70,6 +80,14 @@ export class ReplayProvider implements Provider { ), ); } + case 'transcripts': { + const records = await readTranscriptReplayRecords(source.path); + return requests.map((request) => + transcriptReplayRecordToProviderResponse( + findTranscriptReplayRecord(records, this.lookupForRequest(request)), + ), + ); + } } } @@ -99,7 +117,10 @@ function resolveReplaySource( if (config.fixturesPath) { return { kind: 'fixtures', path: config.fixturesPath }; } + if (config.transcriptsPath) { + return { kind: 'transcripts', path: config.transcriptsPath }; + } throw new Error( - 'Replay provider requires exactly one replay source: fixtures or execution_traces', + 'Replay provider requires exactly one replay source: fixtures, execution_traces, or transcripts', ); } diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index efda8028c..eaa6f4152 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -493,19 +493,6 @@ export interface CopilotSdkResolvedConfig { readonly customProvider?: CopilotCustomProviderConfig; } -export interface CopilotLogResolvedConfig { - /** Explicit path to a session directory containing events.jsonl. */ - readonly sessionDir?: string; - /** Session UUID — combined with sessionStateDir to build the path. */ - readonly sessionId?: string; - /** Auto-discovery mode. 'latest' picks the most recent session. */ - readonly discover?: 'latest'; - /** Override the default ~/.copilot/session-state directory. */ - readonly sessionStateDir?: string; - /** Filter discovery by working directory. */ - readonly cwd?: string; -} - export interface PiCodingAgentResolvedConfig { readonly subprovider?: string; readonly model?: string; @@ -605,6 +592,7 @@ export interface AgentVResolvedConfig { export interface ReplayResolvedConfig { readonly source?: ReplayResolvedSource; readonly fixturesPath?: string; + readonly transcriptsPath?: string; readonly sourceTarget: string; readonly suite?: string; readonly evalPath?: string; @@ -613,7 +601,8 @@ export interface ReplayResolvedConfig { export type ReplayResolvedSource = | { readonly kind: 'fixtures'; readonly path: string } - | { readonly kind: 'execution_traces'; readonly path: string }; + | { readonly kind: 'execution_traces'; readonly path: string } + | { readonly kind: 'transcripts'; readonly path: string }; export interface TargetDeprecationWarning { readonly location: string; @@ -878,10 +867,6 @@ export type ResolvedTarget = readonly kind: 'copilot-cli'; readonly config: CopilotCliResolvedConfig; }) - | (ResolvedTargetBase & { - readonly kind: 'copilot-log'; - readonly config: CopilotLogResolvedConfig; - }) | (ResolvedTargetBase & { readonly kind: 'pi-sdk' | 'pi-coding-agent'; readonly config: PiCodingAgentResolvedConfig; @@ -1115,6 +1100,11 @@ export function resolveTargetDefinition( `Target "${parsed.name}" uses ambiguous provider '${provider}'. Choose an explicit provider such as '${provider}-cli' or '${provider}-sdk'.`, ); } + if (provider === 'copilot-log') { + throw new Error( + `Target "${parsed.name}" uses removed provider 'copilot-log'. Import Copilot events with 'agentv import copilot' and replay the normalized transcript with provider: replay and transcripts: .`, + ); + } const providerBatching = resolveOptionalBoolean(parsed.batch_requests); const subagentModeAllowed = resolveOptionalBoolean(parsed.subagent_mode_allowed); @@ -1186,12 +1176,6 @@ export function resolveTargetDefinition( ...base, config: resolveCopilotCliConfig(parsed, env, evalFilePath), }; - case 'copilot-log': - return { - kind: 'copilot-log', - ...base, - config: resolveCopilotLogConfig(parsed, env), - }; case 'pi-sdk': case 'pi-coding-agent': return { @@ -2481,18 +2465,31 @@ function resolveReplayConfig( allowLiteral: true, }, ); - if ((fixtures ? 1 : 0) + (executionTraces ? 1 : 0) !== 1) { + const transcripts = resolveOptionalString( + target.transcripts, + env, + `${target.name} replay transcripts`, + { + allowLiteral: true, + }, + ); + if ((fixtures ? 1 : 0) + (executionTraces ? 1 : 0) + (transcripts ? 1 : 0) !== 1) { throw new Error( - `Target "${target.name}" (provider: replay) requires exactly one replay source: "fixtures" or "execution_traces"`, + `Target "${target.name}" (provider: replay) requires exactly one replay source: "fixtures", "execution_traces", or "transcripts"`, ); } const fixturesPath = fixtures ? resolveReplaySourcePath(fixtures, evalFilePath) : undefined; const executionTracesPath = executionTraces ? resolveReplaySourcePath(executionTraces, evalFilePath) : undefined; + const transcriptsPath = transcripts + ? resolveReplaySourcePath(transcripts, evalFilePath) + : undefined; const source: ReplayResolvedSource = fixturesPath ? { kind: 'fixtures', path: fixturesPath } - : { kind: 'execution_traces', path: executionTracesPath as string }; + : executionTracesPath + ? { kind: 'execution_traces', path: executionTracesPath } + : { kind: 'transcripts', path: transcriptsPath as string }; const sourceTarget = resolveString( target.source_target, env, @@ -2515,6 +2512,7 @@ function resolveReplayConfig( return { source, fixturesPath, + transcriptsPath, sourceTarget, suite, evalPath, @@ -2719,49 +2717,6 @@ function resolveString( return value; } -function resolveDiscover(value: unknown, targetName: string): 'latest' | undefined { - if (value === undefined || value === null) return undefined; - if (value === 'latest') return 'latest'; - throw new Error(`Target "${targetName}": discover must be "latest" (got "${String(value)}")`); -} - -function resolveCopilotLogConfig( - target: z.infer, - env: EnvLookup, -): CopilotLogResolvedConfig { - const sessionDirSource = target.session_dir; - const sessionIdSource = target.session_id; - const discoverSource = target.discover; - const sessionStateDirSource = target.session_state_dir; - const cwdSource = target.cwd; - - return { - sessionDir: resolveOptionalString( - sessionDirSource, - env, - `${target.name} copilot-log session_dir`, - { allowLiteral: true, optionalEnv: true }, - ), - sessionId: resolveOptionalString( - sessionIdSource, - env, - `${target.name} copilot-log session_id`, - { allowLiteral: true, optionalEnv: true }, - ), - discover: resolveDiscover(discoverSource, target.name), - sessionStateDir: resolveOptionalString( - sessionStateDirSource, - env, - `${target.name} copilot-log session_state_dir`, - { allowLiteral: true, optionalEnv: true }, - ), - cwd: resolveOptionalString(cwdSource, env, `${target.name} copilot-log cwd`, { - allowLiteral: true, - optionalEnv: true, - }), - }; -} - /** * Resolve a string value from targets.yaml, supporting `{{ env.VARIABLE }}` env var syntax. * diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index fd07167ab..6627a0256 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -23,7 +23,6 @@ export type ProviderKind = | 'codex-sdk' | 'copilot-sdk' | 'copilot-cli' - | 'copilot-log' | 'pi-sdk' | 'pi-coding-agent' | 'pi-cli' @@ -42,9 +41,8 @@ export type ProviderKind = * Agent providers that spawn interactive sessions with filesystem access. * These providers read files directly from the filesystem using file:// URIs. * - * Note: copilot-log is intentionally excluded — it is a passive transcript - * reader, not an interactive agent. This allows deterministic-only evals - * (e.g., skill-trigger) to run without a grader_target or LLM API key. + * Passive transcript replay is handled by provider: replay or --transcript, + * not by provider-specific log targets. */ export const AGENT_PROVIDER_KINDS: readonly ProviderKind[] = [ 'codex-cli', @@ -67,7 +65,7 @@ export const AGENT_PROVIDER_KINDS: readonly ProviderKind[] = [ * Used by the orchestrator to decide whether a target can double as its own * grader when no explicit grader_target is configured. * - * Providers NOT in this list (agent providers, transcript, cli, copilot-log) + * Providers NOT in this list (agent providers, transcript, cli, replay) * cannot produce grader responses and should not be used as graders. */ export const LLM_GRADER_CAPABLE_KINDS: readonly ProviderKind[] = [ @@ -95,7 +93,6 @@ export const KNOWN_PROVIDERS: readonly ProviderKind[] = [ 'codex-sdk', 'copilot-sdk', 'copilot-cli', - 'copilot-log', 'pi-sdk', 'pi-coding-agent', 'pi-cli', @@ -493,6 +490,7 @@ export interface TargetDefinition { // Replay fixture fields readonly fixtures?: string | unknown | undefined; readonly execution_traces?: string | unknown | undefined; + readonly transcripts?: string | unknown | undefined; readonly source_target?: string | unknown | undefined; readonly eval_path?: string | unknown | undefined; // VSCode fields @@ -504,11 +502,6 @@ export interface TargetDefinition { readonly attachments_format?: string | unknown | undefined; readonly env?: unknown | undefined; readonly healthcheck?: unknown | undefined; - // Copilot-log fields - readonly session_dir?: string | unknown | undefined; - readonly session_id?: string | unknown | undefined; - readonly discover?: string | unknown | undefined; - readonly session_state_dir?: string | unknown | undefined; // Copilot SDK fields readonly cli_url?: string | unknown | undefined; readonly cli_path?: string | unknown | undefined; diff --git a/packages/core/src/evaluation/replay-transcripts.ts b/packages/core/src/evaluation/replay-transcripts.ts new file mode 100644 index 000000000..30bb415bf --- /dev/null +++ b/packages/core/src/evaluation/replay-transcripts.ts @@ -0,0 +1,82 @@ +/** + * Transcript replay source for provider-agnostic recorded trajectory cassettes. + * + * Imported coding-agent logs are normalized into AgentV transcript JSONL first. + * The replay provider can then substitute those recorded trajectories for a + * live target by matching the current eval case to transcript `test_id` and + * `source_target`, while graders run fresh over AgentV Message[] output. + */ + +import { + type TranscriptReplayEntry, + groupTranscriptJsonLines, + readTranscriptJsonl, +} from '../import/types.js'; +import type { ProviderResponse } from './providers/types.js'; +import type { ReplayFixtureLookup } from './replay-fixtures.js'; + +export interface TranscriptReplayRecord { + readonly entry: TranscriptReplayEntry; + readonly sourcePath: string; +} + +export async function readTranscriptReplayRecords( + sourcePath: string, +): Promise { + const lines = await readTranscriptJsonl(sourcePath); + return groupTranscriptJsonLines(lines).map((entry) => ({ entry, sourcePath })); +} + +export function findTranscriptReplayRecord( + records: readonly TranscriptReplayRecord[], + lookup: ReplayFixtureLookup, +): TranscriptReplayRecord { + const matches = records.filter((record) => transcriptRecordMatches(record.entry, lookup)); + if (matches.length === 1) { + return matches[0]; + } + + const key = `test_id=${lookup.testId} source_target=${lookup.sourceTarget}`; + if (matches.length === 0) { + throw new Error(`Transcript replay lookup found no record for ${key}`); + } + throw new Error(`Transcript replay lookup found ${matches.length} duplicate records for ${key}`); +} + +export function transcriptReplayRecordToProviderResponse( + record: TranscriptReplayRecord, +): ProviderResponse { + const entry = record.entry; + return { + output: entry.messages, + tokenUsage: entry.tokenUsage, + durationMs: entry.durationMs, + costUsd: entry.costUsd ?? undefined, + startTime: entry.source.startedAt, + raw: { + replay_transcript: dropUndefined({ + source_path: record.sourcePath, + test_id: entry.testId, + target: entry.target, + source_provider: entry.source.provider, + source_session_id: entry.source.sessionId, + source_kind: entry.source.kind, + source_format: entry.source.format, + source_model: entry.source.model, + source_cwd: entry.source.cwd, + source_metadata: entry.source.metadata, + }), + }, + }; +} + +function transcriptRecordMatches( + entry: TranscriptReplayEntry, + lookup: ReplayFixtureLookup, +): boolean { + return entry.testId === lookup.testId && entry.target === lookup.sourceTarget; +} + +function dropUndefined(value: Record): Record { + return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined)); +} diff --git a/packages/core/src/evaluation/transcript-summary.ts b/packages/core/src/evaluation/transcript-summary.ts index 9fea5d35f..d37083522 100644 --- a/packages/core/src/evaluation/transcript-summary.ts +++ b/packages/core/src/evaluation/transcript-summary.ts @@ -47,7 +47,6 @@ const PROVIDER_ALIASES: Readonly> = { copilot: 'copilot-sdk', 'copilot-cli': 'copilot-cli', 'copilot-sdk': 'copilot-sdk', - 'copilot-log': 'copilot-log', pi: 'pi-cli', 'pi-cli': 'pi-cli', 'pi-rpc': 'pi-rpc', diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index 802ab066e..4f6231c0b 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -250,6 +250,7 @@ const REPLAY_SETTINGS = new Set([ ...COMMON_SETTINGS, 'fixtures', 'execution_traces', + 'transcripts', 'source_target', 'suite', 'eval_path', @@ -730,13 +731,16 @@ export async function validateTargetsFile(filePath: string): Promise; private cursor = 0; constructor(targetName: string, entries: TranscriptReplayEntry[]) { this.targetName = targetName; this.id = `transcript:${targetName}`; this.entries = entries; + this.entriesByTestId = new Map(entries.map((entry) => [entry.testId, entry])); } /** @@ -49,15 +52,14 @@ export class TranscriptProvider implements Provider { return this.entries.length; } - async invoke(_request: ProviderRequest): Promise { - if (this.cursor >= this.entries.length) { - throw new Error( - `Transcript exhausted: ${this.entries.length} entr${this.entries.length === 1 ? 'y' : 'ies'} available but ` + - `${this.cursor + 1} invocations attempted. Each transcript entry maps to one test case.`, - ); - } + get testIds(): readonly string[] { + return this.entries.map((entry) => entry.testId); + } - const entry = this.entries[this.cursor++]; + async invoke(request: ProviderRequest): Promise { + const entry = request.evalCaseId + ? this.entryForTestId(request.evalCaseId) + : this.nextPositionalEntry(); return { output: entry.messages, @@ -74,4 +76,25 @@ export class TranscriptProvider implements Provider { startTime: entry.source.startedAt, }; } + + private entryForTestId(testId: string): TranscriptReplayEntry { + const entry = this.entriesByTestId.get(testId); + if (entry) { + return entry; + } + throw new Error( + `Transcript replay found no entry for test_id=${testId}. Available test_id values: ${this.testIds.join(', ') || ''}`, + ); + } + + private nextPositionalEntry(): TranscriptReplayEntry { + if (this.cursor >= this.entries.length) { + throw new Error( + `Transcript exhausted: ${this.entries.length} entr${this.entries.length === 1 ? 'y' : 'ies'} available but ` + + `${this.cursor + 1} invocations attempted. Each transcript entry maps to one test case.`, + ); + } + + return this.entries[this.cursor++]; + } } diff --git a/packages/core/src/import/types.ts b/packages/core/src/import/types.ts index 989b7d969..45a916833 100644 --- a/packages/core/src/import/types.ts +++ b/packages/core/src/import/types.ts @@ -927,6 +927,7 @@ export function groupTranscriptJsonLines( const grouped = new Map< string, { + testId: string; target: string; tokenUsage?: ProviderTokenUsage; durationMs?: number; @@ -937,7 +938,8 @@ export function groupTranscriptJsonLines( >(); for (const line of lines) { - const existing = grouped.get(line.test_id); + const groupKey = transcriptGroupKey(line.test_id, line.target); + const existing = grouped.get(groupKey); const source: TranscriptSource = { kind: line.source.kind, provider: line.source.provider, @@ -965,7 +967,8 @@ export function groupTranscriptJsonLines( continue; } - grouped.set(line.test_id, { + grouped.set(groupKey, { + testId: line.test_id, target: line.target, tokenUsage: transcriptTokenUsage, durationMs: line.transcript_duration_ms, @@ -975,8 +978,8 @@ export function groupTranscriptJsonLines( }); } - return [...grouped.entries()].map(([testId, entry]) => ({ - testId, + return [...grouped.values()].map((entry) => ({ + testId: entry.testId, target: entry.target, tokenUsage: entry.tokenUsage, durationMs: entry.durationMs, @@ -988,6 +991,10 @@ export function groupTranscriptJsonLines( })); } +function transcriptGroupKey(testId: string, target: string): string { + return `${testId}\u0000${target}`; +} + /** * Read a transcript JSONL file and parse each line into a TranscriptJsonLine. */ diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e4f044424..793eb4ba8 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -10,6 +10,7 @@ export * from './evaluation/external-trace.js'; export * from './evaluation/projection-identity.js'; export * from './evaluation/replay-fixtures.js'; export * from './evaluation/replay-trace-envelopes.js'; +export * from './evaluation/replay-transcripts.js'; export { ResultRowSchemaError, normalizeResultRow, diff --git a/packages/core/test/evaluation/providers/copilot-log.test.ts b/packages/core/test/evaluation/providers/copilot-log.test.ts deleted file mode 100644 index 9e4f36dcb..000000000 --- a/packages/core/test/evaluation/providers/copilot-log.test.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import path from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { CopilotLogProvider } from '../../../src/evaluation/providers/copilot-log.js'; - -/** Build a JSONL event line with data nesting matching real Copilot CLI format. */ -function eventLine(type: string, data: Record = {}): string { - return JSON.stringify({ type, data, id: 'evt-1', timestamp: '2026-03-25T10:00:00.000Z' }); -} - -describe('CopilotLogProvider', () => { - let tempDir: string; - - beforeEach(async () => { - tempDir = await mkdtemp(path.join(tmpdir(), 'copilot-log-provider-')); - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - async function createSession(sessionId: string, events: string) { - const sessionDir = path.join(tempDir, sessionId); - await mkdir(sessionDir, { recursive: true }); - await writeFile(path.join(sessionDir, 'workspace.yaml'), 'cwd: /projects/app\n'); - await writeFile(path.join(sessionDir, 'events.jsonl'), events); - return sessionDir; - } - - it('reads transcript from explicit session_dir', async () => { - const sessionDir = await createSession( - 's1', - [ - eventLine('user.message', { content: 'hello' }), - eventLine('assistant.message', { content: 'hi', toolRequests: [] }), - ].join('\n'), - ); - - const provider = new CopilotLogProvider('test', { sessionDir }); - const response = await provider.invoke({ question: 'ignored' }); - - expect(response.output).toBeDefined(); - expect(response.output?.length).toBeGreaterThan(0); - expect(response.output?.[0].role).toBe('user'); - expect(response.output?.[0].content).toBe('hello'); - }); - - it('reads transcript from session_id + session_state_dir', async () => { - await createSession( - 'uuid-abc', - [eventLine('user.message', { content: 'test input' })].join('\n'), - ); - - const provider = new CopilotLogProvider('test', { - sessionId: 'uuid-abc', - sessionStateDir: tempDir, - }); - const response = await provider.invoke({ question: 'ignored' }); - - expect(response.output).toBeDefined(); - expect(response.output?.[0].content).toBe('test input'); - }); - - it('auto-discovers latest session with discover=latest', async () => { - await createSession('uuid-old', [eventLine('user.message', { content: 'old' })].join('\n')); - await new Promise((r) => setTimeout(r, 50)); - await createSession('uuid-new', [eventLine('user.message', { content: 'new' })].join('\n')); - - const provider = new CopilotLogProvider('test', { - discover: 'latest', - sessionStateDir: tempDir, - }); - const response = await provider.invoke({ question: 'ignored' }); - - expect(response.output).toBeDefined(); - expect(response.output?.[0].content).toBe('new'); - }); - - it('returns token usage from session.shutdown modelMetrics', async () => { - const sessionDir = await createSession( - 's1', - [ - eventLine('session.shutdown', { - shutdownType: 'normal', - currentModel: 'gpt-4o', - modelMetrics: { - 'gpt-4o': { usage: { inputTokens: 500, outputTokens: 200 } }, - }, - }), - ].join('\n'), - ); - - const provider = new CopilotLogProvider('test', { sessionDir }); - const response = await provider.invoke({ question: 'ignored' }); - - expect(response.tokenUsage).toEqual({ input: 500, output: 200 }); - }); - - it('throws when no session found', async () => { - const provider = new CopilotLogProvider('test', { - sessionId: 'nonexistent', - sessionStateDir: tempDir, - }); - - await expect(provider.invoke({ question: 'x' })).rejects.toThrow(); - }); - - it('has correct provider metadata', () => { - const provider = new CopilotLogProvider('my-target', { sessionDir: '/tmp/s1' }); - expect(provider.id).toBe('copilot-log:my-target'); - expect(provider.kind).toBe('copilot-log'); - expect(provider.targetName).toBe('my-target'); - }); -}); diff --git a/packages/core/test/evaluation/providers/normalize-tool-call.test.ts b/packages/core/test/evaluation/providers/normalize-tool-call.test.ts index 65215e49a..7ccf99e1e 100644 --- a/packages/core/test/evaluation/providers/normalize-tool-call.test.ts +++ b/packages/core/test/evaluation/providers/normalize-tool-call.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from 'vitest'; -import { normalizeToolCall } from '../../../src/evaluation/providers/normalize-tool-call.js'; +import { + type ToolNormalizationSource, + normalizeToolCall, +} from '../../../src/evaluation/providers/normalize-tool-call.js'; import type { ProviderKind } from '../../../src/evaluation/providers/types.js'; import type { ToolCall } from '../../../src/evaluation/providers/types.js'; @@ -49,10 +52,10 @@ describe('normalizeToolCall', () => { for (const provider of [ 'copilot-cli', 'copilot-sdk', - 'copilot-log', + 'copilot-events', 'vscode', 'vscode-insiders', - ] as ProviderKind[]) { + ] as ToolNormalizationSource[]) { it(`${provider}: skill (lowercase) → Skill`, () => { const result = normalizeToolCall(provider, tc('skill', { skill: 'my-skill' })); expect(result.tool).toBe('Skill'); diff --git a/packages/core/test/evaluation/providers/replay-transcripts.test.ts b/packages/core/test/evaluation/providers/replay-transcripts.test.ts new file mode 100644 index 000000000..8473384a4 --- /dev/null +++ b/packages/core/test/evaluation/providers/replay-transcripts.test.ts @@ -0,0 +1,120 @@ +import { afterEach, describe, expect, it } from 'bun:test'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { ReplayProvider, type TranscriptEntry, toTranscriptJsonLines } from '../../../src/index.js'; + +describe('ReplayProvider transcript source', () => { + const tempDirs: string[] = []; + + afterEach(async () => { + await Promise.all(tempDirs.map((dir) => rm(dir, { recursive: true, force: true }))); + tempDirs.length = 0; + }); + + it('replays normalized transcript JSONL by test_id and source_target', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'agentv-replay-transcript-')); + tempDirs.push(dir); + const transcriptPath = path.join(dir, 'transcript.jsonl'); + const transcript: TranscriptEntry = { + messages: [ + { role: 'user', content: 'Inspect the repo' }, + { + role: 'assistant', + content: 'I inspected it.', + toolCalls: [{ tool: 'Read', input: { path: 'package.json' }, output: '{}' }], + }, + ], + source: { + kind: 'imported_transcript', + provider: 'copilot', + sessionId: 'copilot-session-1', + model: 'gpt-5-mini', + }, + tokenUsage: { input: 10, output: 5 }, + durationMs: 1234, + }; + const rows = toTranscriptJsonLines(transcript, { + testId: 'copilot-case', + target: 'copilot-cli', + }); + await writeFile(transcriptPath, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`); + + const provider = new ReplayProvider('copilot-cassette', { + source: { kind: 'transcripts', path: transcriptPath }, + sourceTarget: 'copilot-cli', + }); + const response = await provider.invoke({ + question: 'ignored', + evalCaseId: 'copilot-case', + }); + + expect(response.output).toEqual(transcript.messages); + expect(response.tokenUsage).toEqual({ input: 10, output: 5 }); + expect(response.durationMs).toBe(1234); + expect(response.raw?.replay_transcript).toMatchObject({ + test_id: 'copilot-case', + target: 'copilot-cli', + source_provider: 'copilot', + source_session_id: 'copilot-session-1', + }); + }); + + it('does not replay a transcript for the wrong test_id', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'agentv-replay-transcript-')); + tempDirs.push(dir); + const transcriptPath = path.join(dir, 'transcript.jsonl'); + const rows = toTranscriptJsonLines( + { + messages: [{ role: 'assistant', content: 'Recorded answer' }], + source: { provider: 'copilot', sessionId: 'copilot-session-1' }, + }, + { testId: 'recorded-case', target: 'copilot-cli' }, + ); + await writeFile(transcriptPath, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`); + + const provider = new ReplayProvider('copilot-cassette', { + source: { kind: 'transcripts', path: transcriptPath }, + sourceTarget: 'copilot-cli', + }); + + await expect( + provider.invoke({ question: 'ignored', evalCaseId: 'different-case' }), + ).rejects.toThrow(/Transcript replay lookup found no record for test_id=different-case/); + }); + + it('matches source_target when one transcript file has the same test_id for multiple targets', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'agentv-replay-transcript-')); + tempDirs.push(dir); + const transcriptPath = path.join(dir, 'transcript.jsonl'); + const rows = [ + ...toTranscriptJsonLines( + { + messages: [{ role: 'assistant', content: 'Copilot answer' }], + source: { provider: 'copilot', sessionId: 'copilot-session-1' }, + }, + { testId: 'shared-case', target: 'copilot-cli' }, + ), + ...toTranscriptJsonLines( + { + messages: [{ role: 'assistant', content: 'Claude answer' }], + source: { provider: 'claude', sessionId: 'claude-session-1' }, + }, + { testId: 'shared-case', target: 'claude-cli' }, + ), + ]; + await writeFile(transcriptPath, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`); + + const provider = new ReplayProvider('claude-cassette', { + source: { kind: 'transcripts', path: transcriptPath }, + sourceTarget: 'claude-cli', + }); + const response = await provider.invoke({ + question: 'ignored', + evalCaseId: 'shared-case', + }); + + expect(response.output?.[0]?.content).toBe('Claude answer'); + }); +}); diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index 4ab98c297..8a1d8a093 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -342,6 +342,19 @@ describe('resolveTargetDefinition', () => { ).toThrow(/judge_target.*has been removed/i); }); + it('rejects removed copilot-log target provider surface', () => { + expect(() => + resolveTargetDefinition( + { + name: 'old-copilot-log', + provider: 'copilot-log', + discover: 'latest', + } as never, + {}, + ), + ).toThrow(/removed provider 'copilot-log'.*agentv import copilot.*provider: replay/i); + }); + it('rejects removed log_format target aliases', () => { expect(() => resolveTargetDefinition( diff --git a/packages/core/test/evaluation/validation/targets-validator.test.ts b/packages/core/test/evaluation/validation/targets-validator.test.ts index 6bb6e99c1..481f7fb13 100644 --- a/packages/core/test/evaluation/validation/targets-validator.test.ts +++ b/packages/core/test/evaluation/validation/targets-validator.test.ts @@ -588,6 +588,26 @@ targets: ).toBe(false); }); + it('accepts replay targets backed by normalized transcripts', async () => { + const filePath = path.join(tempDir, 'replay-transcripts.yaml'); + await writeFile( + filePath, + `targets: + - id: replay-transcript + provider: replay + transcripts: ./fixtures/transcript.jsonl + source_target: live-agent +`, + ); + + const result = await validateTargetsFile(filePath); + + expect(result.valid).toBe(true); + expect( + result.errors.some((error) => error.message.includes("Unknown setting 'transcripts'")), + ).toBe(false); + }); + it('rejects replay targets with ambiguous source configuration', async () => { const filePath = path.join(tempDir, 'replay-ambiguous-source.yaml'); await writeFile( diff --git a/packages/core/test/import/transcript-provider.test.ts b/packages/core/test/import/transcript-provider.test.ts index 5ed8eb466..47975f1a6 100644 --- a/packages/core/test/import/transcript-provider.test.ts +++ b/packages/core/test/import/transcript-provider.test.ts @@ -64,6 +64,70 @@ describe('TranscriptProvider', () => { expect(response.startTime).toBe('2026-03-13T00:00:00.000Z'); }); + it('matches replay entries by evalCaseId/test_id instead of file position', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'agentv-transcript-provider-')); + tempDirs.push(dir); + const transcriptPath = path.join(dir, 'transcript.jsonl'); + + const first = toTranscriptJsonLines( + { + messages: [ + { role: 'user', content: 'First task' }, + { role: 'assistant', content: 'First answer' }, + ], + source: { provider: 'claude', sessionId: 'one' }, + }, + { testId: 'case-a', target: 'claude' }, + ); + const second = toTranscriptJsonLines( + { + messages: [ + { role: 'user', content: 'Second task' }, + { role: 'assistant', content: 'Second answer' }, + ], + source: { provider: 'claude', sessionId: 'two' }, + }, + { testId: 'case-b', target: 'claude' }, + ); + + await writeFile( + transcriptPath, + `${[...first, ...second].map((line) => JSON.stringify(line)).join('\n')}\n`, + 'utf8', + ); + + const provider = await TranscriptProvider.fromFile(transcriptPath); + const response = await provider.invoke({ question: 'ignored', evalCaseId: 'case-b' }); + + expect(provider.testIds).toEqual(['case-a', 'case-b']); + expect(response.output?.[1]?.content).toBe('Second answer'); + }); + + it('fails loudly when no transcript test_id matches the eval case', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'agentv-transcript-provider-')); + tempDirs.push(dir); + const transcriptPath = path.join(dir, 'transcript.jsonl'); + const lines = toTranscriptJsonLines( + { + messages: [{ role: 'assistant', content: 'Recorded answer' }], + source: { provider: 'claude', sessionId: 'one' }, + }, + { testId: 'recorded-case', target: 'claude' }, + ); + + await writeFile( + transcriptPath, + `${lines.map((line) => JSON.stringify(line)).join('\n')}\n`, + 'utf8', + ); + + const provider = await TranscriptProvider.fromFile(transcriptPath); + + await expect( + provider.invoke({ question: 'ignored', evalCaseId: 'different-case' }), + ).rejects.toThrow(/no entry for test_id=different-case.*recorded-case/); + }); + it('counts distinct test transcripts instead of raw JSONL rows', async () => { const dir = await mkdtemp(path.join(tmpdir(), 'agentv-transcript-provider-')); tempDirs.push(dir);