diff --git a/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx b/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx index 21b577a1f..3011d167c 100644 --- a/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx +++ b/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx @@ -10,12 +10,16 @@ AgentV writes each eval invocation as a portable run bundle. The bundle is the source of truth for Dashboard, reports, compare/trend tooling, CI gates, and external adapters. -The contract is run-centric: - -- `summary.json` owns aggregate run facts. -- `.internal/index.jsonl` owns per-run row discovery and filtering. -- Per-case sidecars own detailed payloads such as grading, metrics, transcripts, - generated files, and raw provider evidence. +The contract is run-centric, and each layer answers a different question: + +- `summary.json` answers **run aggregate** questions: status breakdown, per-test + and per-sample counts, token/cost usage, and rollups for the whole run. +- `.internal/index.jsonl` answers **dashboard-ready row manifest** questions: + identity, filter metadata, status, scores, and explicit sidecar paths for + every test and sample in the run. +- Per-sample sidecars answer **detailed evidence** questions: full grading + breakdowns, metrics, transcripts, generated files, and raw provider evidence + for one sample. - Dashboard, search, SQLite, HTML reports, and vendor exports are rebuildable projections over the bundle. @@ -95,9 +99,9 @@ reserved for rebuildable local state and are skipped by run discovery. | File or field | Owns | Use it for | | --- | --- | --- | -| `summary.json` | Aggregate run metadata and rollups: run id, experiment label, tags, runtime source, counts, pass rate, score summaries, duration, token/cost totals, and writer metadata. | Listing runs, CI summaries, quick dashboards, trend cards, and validating that a run is complete enough to inspect. | -| `.internal/index.jsonl` | Canonical per-run row index: one row per case/result aggregate, with identity fields, filter metadata, scores, status, and explicit run-relative paths to sidecars. | Filtering, compare/trend inputs, Dashboard detail routing, rerun/resume lookup, export adapters, and artifact discovery. | -| `result.json` | Compact per-sample manifest for one `sample-N/` directory, including AgentV `execution_status` and normalized outcome `status`. | Loading one sample without scanning the whole run index. | +| `summary.json` | Aggregate run metadata and rollups: run id, experiment label, tags, runtime source, `counts` (`total_tests`/`passed_tests`/`failed_tests`/`total_samples`/`errored_samples`), pass rate, per-test rollups in `tests[]`, score summaries, duration, token/cost totals, and writer metadata. | Listing runs, CI summaries, quick dashboards, trend cards, and validating that a run is complete enough to inspect. | +| `.internal/index.jsonl` | Canonical per-run row index: one dashboard-ready row per test/result, with identity fields, filter metadata, scores, status, a compact `target_error_kind`, and explicit run-relative paths to sidecars such as `target_execution_path` and `metrics_path`. | Filtering, compare/trend inputs, Dashboard detail routing, rerun/resume lookup, export adapters, and artifact discovery. | +| `result.json` | Compact per-sample manifest for one `sample-N/` directory, including AgentV `execution_status`, normalized outcome `status`, the full target runtime envelope, and `transcript_summary`. | Loading one sample's full detail without scanning the whole run index or a separate sidecar. | | `environment.json` / `environment_path` | Redacted environment recipe provenance: authored inline/file reference, resolved recipe hash, host or Docker type, resolved workdir, setup argv command, setup log output/error, Docker context/image/digest fields when available, and repo provenance only when authored or emitted by setup. Index rows carry `environment_path` plus a compact `environment` summary; large setup logs stay in the sidecar. | Reproducing and reviewing the testbed without treating setup side effects as row metadata. Repository identity is opaque unless the environment recipe or setup output states it explicitly. | | `grading.json` | Grader outputs and scoring provenance: aggregate `pass`, `score`, `reason`, recursive `component_results`, and optional `assertion`, `named_scores`, and `metadata`. | Explaining why a row passed or failed. | | `metrics.json` | Duration, token usage, cost, execution status, trajectory, and derived executor behavior such as tool calls, files touched, shell commands, errors, turns, and output sizes. | Dashboard behavior views, cost/latency reporting, metric-style graders, adapter projections, and lightweight analysis. | @@ -115,6 +119,116 @@ a row reader should not parse aggregate summary structures to find one case's grading or transcript. Keep aggregate questions on `summary.json`; keep row and artifact discovery on `.internal/index.jsonl`. +## Summary Contract + +`summary.json` answers run-aggregate questions only. It reports counts and +rollups at both the test level (one entry per `eval_path` + `test_id` + +`target` combination) and the sample level (one execution, including repeats), +plus usage, infra-failure taxonomy, and writer metadata. It does not inline +row-level identity, scores, or sidecar paths — that is `.internal/index.jsonl`'s +job. + +```json +{ + "index_path": ".internal/index.jsonl", + "run_id": "2026-06-30T08-15-00-000Z", + "status": { + "passed": { "count": 1, "percentage": 50 }, + "failed": { "count": 1, "percentage": 50 }, + "errored": { "count": 1, "percentage": 50 }, + "skipped": { "count": 0, "percentage": 0 } + }, + "counts": { + "total_tests": 2, + "passed_tests": 1, + "failed_tests": 1, + "total_samples": 2, + "errored_samples": 1 + }, + "usage": { + "total_tokens": 4200, + "input_tokens": 3100, + "output_tokens": 1100, + "reasoning_tokens": 0, + "cost_usd": 0.014 + }, + "infra_failures": { + "total": 1, + "reasons": [{ "reason": "execution_error", "count": 1 }] + }, + "tests": [ + { + "test_id": "refund-eligibility", + "suite": "support", + "eval_path": "evals/support/refunds.eval.yaml", + "target": "codex-gpt5", + "total_samples": 1, + "passed_samples": 1, + "status_counts": { "passed": 1 }, + "pass_rate": 100, + "pass_any": true, + "samples": [ + { + "test_id": "refund-eligibility", + "target": "codex-gpt5", + "sample_index": 1, + "status": "passed", + "score": 0.92, + "execution_status": "ok", + "duration_ms": 184200 + } + ] + }, + { + "test_id": "refund-timeout", + "suite": "support", + "eval_path": "evals/support/refunds.eval.yaml", + "target": "codex-gpt5", + "total_samples": 1, + "passed_samples": 0, + "status_counts": { "execution_error": 1 }, + "pass_rate": 0, + "pass_any": false, + "samples": [ + { + "test_id": "refund-timeout", + "target": "codex-gpt5", + "sample_index": 1, + "status": "execution_error", + "score": 0, + "execution_status": "execution_error", + "duration_ms": 300000 + } + ] + } + ], + "metadata": { + "eval_file": "evals/support/refunds.eval.yaml", + "timestamp": "2026-06-30T08:15:00.000Z", + "targets": ["codex-gpt5"], + "tests_run": ["refund-eligibility", "refund-timeout"], + "experiment": "with_skills", + "tags": { "experiment": "with_skills", "team": "support" } + }, + "run_summary": { + "codex-gpt5": { + "pass_rate": { "mean": 50, "stddev": 0 }, + "time_seconds": { "mean": 184.2, "stddev": 0 }, + "tokens": { "mean": 2100, "stddev": 0 } + } + }, + "metrics": { "duration": {}, "tokens": {}, "cost": {} }, + "notes": [] +} +``` + +`run_id` lives at the summary root, not under `metadata`. `counts` reports +test-level totals (`total_tests`/`passed_tests`/`failed_tests`) alongside +sample-level totals (`total_samples`/`errored_samples`), since one test can +produce multiple samples through repeats. `tests[]` is the per-test rollup +array; row-level identity, scores, and sidecar paths live in +`.internal/index.jsonl`, not here. + ## Grading Contract Each per-sample `grading.json` uses Promptfoo-compatible grading result @@ -194,6 +308,16 @@ adds providers and projections, but stable rows follow these rules: - Field names are `snake_case`. - Identity and filter fields live on the row, not only in directory names. - Sidecar references are explicit path fields, relative to the run directory. +- Detailed payloads stay in sidecars: the row carries a compact + `target_error_kind` classification plus `target_execution_path`, not the full + target runtime envelope; transcript detail lives in `transcript_path` and + each sample's `result.json`, not inlined on the row. +- `projection_identity` is the one detailed-looking field that stays inline by + design rather than moving to a sidecar. `writeArtifactsFromResults` reads + `projection_identity.id` back off previously written rows to decide + skip/update/error duplicate policy across separate `agentv eval` invocations + appending to the same run; a sidecar would force an extra file read per row + on every append. - Run-level provenance such as `runtime_source` belongs in `summary.json`, not repeated on every row. It records runtime config provenance and eval file paths; the experiment label remains on `experiment` and `tags.experiment`. @@ -207,7 +331,6 @@ Example row: ```json { "timestamp": "2026-06-30T08:15:00.000Z", - "run_id": "2026-06-30T08-15-00-000Z", "experiment": "with_skills", "tags": { "experiment": "with_skills", "team": "support" }, "eval_path": "evals/support/refunds.eval.yaml", @@ -227,31 +350,41 @@ Example row: "target_execution_path": "refund-eligibility--4f9a7c2d1b6e/sample-1/target-execution.json", "stdout_path": "refund-eligibility--4f9a7c2d1b6e/sample-1/stdout.txt", "stderr_path": "refund-eligibility--4f9a7c2d1b6e/sample-1/stderr.txt", - "target_execution": { - "schema_version": "agentv.target_execution.v1", - "status": "success", - "provider_kind": "cli", - "target_id": "codex-gpt5" - }, "transcript_path": "refund-eligibility--4f9a7c2d1b6e/sample-1/transcript.json", "transcript_raw_path": "refund-eligibility--4f9a7c2d1b6e/sample-1/transcript-raw.jsonl", - "transcript_summary": { - "total_turns": 4, - "tool_calls": { "file_read": 2, "shell": 1, "unknown": 0 }, - "files_read": ["src/refunds.ts"], - "files_modified": ["src/refunds.ts"], - "shell_commands": ["bun test refunds.test.ts"], - "web_fetches": [], - "errors": [], - "thinking_blocks": 1 - }, "output_path": "refund-eligibility--4f9a7c2d1b6e/sample-1/outputs/answer.md", "answer_path": "refund-eligibility--4f9a7c2d1b6e/sample-1/outputs/answer.md", "file_changes_path": "refund-eligibility--4f9a7c2d1b6e/sample-1/outputs/file_changes.diff", - "test_dir": "refund-eligibility--4f9a7c2d1b6e/test" + "test_dir": "refund-eligibility--4f9a7c2d1b6e/test", + "projection_identity": { + "schema_version": "agentv.projection_identity.v1", + "id": "agentv-prj-9f1c2e7a4b6d80519f1c2e7a4b6d8051", + "key": "agentv.projection_identity.v1:projection_format=agentv%2Fartifacts%2Fv1|projection_version=1|run_id=2026-06-30T08-15-00-000Z|suite=~|eval_path=evals%2Fsupport%2Frefunds.eval.yaml|test_id=refund-eligibility|target=codex-gpt5|source_target=codex-gpt5|attempt=0|variant=skills-v2|envelope_id=envelope-9f1c2e7a|trace_id=trace-4b6d8051|root_span_id=span-1a2b3c4d", + "dimensions": { + "run_id": "2026-06-30T08-15-00-000Z", + "eval_path": "evals/support/refunds.eval.yaml", + "test_id": "refund-eligibility", + "target": "codex-gpt5", + "source_target": "codex-gpt5", + "attempt": 0, + "variant": "skills-v2", + "envelope_id": "envelope-9f1c2e7a", + "trace_id": "trace-4b6d8051", + "root_span_id": "span-1a2b3c4d", + "projection_format": "agentv/artifacts/v1", + "projection_version": "1" + } + } } ``` +Full target-runtime detail for this row lives at `target_execution_path`; full +transcript detail lives at `transcript_path` and the sample's `result.json`. +This row omits `target_error_kind` because the sample completed without a +target-runtime error; failed target runs carry an error-kind string such as +`signal_crash` or `timeout` on that field instead of requiring a sidecar read +to know a run failed. + Rows can represent repeated samples, multi-target runs, imported suites, manual `prepare`/`grade` samples, or imported provider sessions. That is why `experiment`, `eval_path`, `test_id`, `target`, `variant`, `sample_index`, `retry_index`, and diff --git a/apps/web/src/content/docs/docs/next/tools/wip-checkpoints.mdx b/apps/web/src/content/docs/docs/next/tools/wip-checkpoints.mdx index e430f5bd9..f466ff4d1 100644 --- a/apps/web/src/content/docs/docs/next/tools/wip-checkpoints.mdx +++ b/apps/web/src/content/docs/docs/next/tools/wip-checkpoints.mdx @@ -23,7 +23,7 @@ If no results repo is configured, or auto-push is disabled, `agentv eval` still | Location | Path or ref | What it contains | | --- | --- | --- | -| Local project | `.agentv/results//summary.json` | A run-start stub with `metadata.run_id`, `metadata.experiment`, `metadata.planned_test_count`, and the eval file path when known. This lets Dashboard recognize incomplete local runs as resumable. | +| Local project | `.agentv/results//summary.json` | A run-start stub with `run_id`, `metadata.experiment`, `metadata.planned_test_count`, and the eval file path when known. This lets Dashboard recognize incomplete local runs as resumable. | | Local project | `.agentv/results//.internal/index.jsonl` | Result rows appended as test cases finish. Rows use the normal snake_case result JSONL format. | | Results repo remote | `agentv/wip//` | A forced-updated branch containing the checkpointed run under `.agentv/results//`. | | Results repo storage branch | Configured `results.repo.branch`; local checkout configs default to `agentv/results/v1` | The final published run after `agentv eval` completes and the normal auto-export succeeds. | diff --git a/docs/adr/0011-result-output-artifact-contract.md b/docs/adr/0011-result-output-artifact-contract.md index 9856f233d..155490927 100644 --- a/docs/adr/0011-result-output-artifact-contract.md +++ b/docs/adr/0011-result-output-artifact-contract.md @@ -23,8 +23,12 @@ artifact-format v2 runs direct children of `.agentv/results/` and treats Refined and superseded for current output layout by [ADR 0017](0017-output-artifact-and-workspace-resolver-contract.md). Keep this ADR as historical context; current run bundles use `.internal/index.jsonl`, -`sample-N/`, `metrics.json`, and `grading.json.assertion_results`, not root -`index.jsonl`, `run-N/`, `timing_path`, or `timing.json` as the active contract. +`sample-N/`, `metrics.json`, and `grading.json.component_results`, not root +`index.jsonl`, `run-N/`, `timing_path`, `timing.json`, or +`grading.json.assertion_results` as the active contract. ADR 0017's "Summary/index/sidecar +boundary is locked" amendment (tracker `av-cpl5`) further renames this ADR's +`summary.json` `cases`/`case` vocabulary to `tests`/`test`, and moves +`index.jsonl`'s row-level target-execution/transcript detail into sidecars. ## Context diff --git a/docs/adr/0012-finalize-run-artifact-layout.md b/docs/adr/0012-finalize-run-artifact-layout.md index 69d110bfd..98620cb1a 100644 --- a/docs/adr/0012-finalize-run-artifact-layout.md +++ b/docs/adr/0012-finalize-run-artifact-layout.md @@ -15,7 +15,12 @@ Refined and superseded for current output layout by [ADR 0017](0017-output-artifact-and-workspace-resolver-contract.md). Keep this ADR as historical context for the results-root move; current run bundles place the per-run index at `.internal/index.jsonl`, store repeated executions under -`sample-N/`, and merge timing data into `metrics.json`. +`sample-N/`, and merge timing data into `metrics.json`. ADR 0017's +"Summary/index/sidecar boundary is locked" amendment (tracker `av-cpl5`) +further renames this ADR's `summary.json` counts/array vocabulary from +`cases`/`case` to `tests`/`test`/`sample`, and moves `index.jsonl`'s row-level +`target_execution`/`transcript_summary` detail into sidecars behind a compact +`target_error_kind` field. ## Context diff --git a/docs/adr/0017-output-artifact-and-workspace-resolver-contract.md b/docs/adr/0017-output-artifact-and-workspace-resolver-contract.md index fc35b2190..a93a84eea 100644 --- a/docs/adr/0017-output-artifact-and-workspace-resolver-contract.md +++ b/docs/adr/0017-output-artifact-and-workspace-resolver-contract.md @@ -23,6 +23,12 @@ contract below supersedes both the earlier agentskills-shaped `graders[]`/`checks[]` wording. Native AgentV grading artifacts now use a recursive Promptfoo-style grading result in `snake_case`. +Amended (2026-07-06) by tracker `av-cpl5`: decision point 3 below described +`transcript_summary` as inlined into each `index.jsonl` result row. That +inlining is superseded — see "Summary/index/sidecar boundary is locked" below +for the current contract and the rationale for what stays inline versus what +moves to a sidecar. + ## Context We reviewed the output formats of promptfoo, margin-lab, vercel-agent-eval, and @@ -107,6 +113,59 @@ protocol payloads. AgentV wrappers around those payloads still use `snake_case`. count, usage, runtime, case, and failure summaries; add promptfoo-shaped `named_scores`/`derived_metrics` on rows. +### Summary/index/sidecar boundary is locked (tracker `av-cpl5`) + +Tracker `av-cpl5` closes out the boundary point 2 and point 3 above left open: +exactly what stays on the `summary.json` root, what stays inline on an +`index.jsonl` row, and what must live only in a per-sample sidecar. + +- **`summary.json` uses `tests`/`test` terminology, not `cases`/`case`** + (`av-cpl5.2`). `counts.total_cases`/`passed_cases`/`failed_cases` are renamed + to `counts.total_tests`/`passed_tests`/`failed_tests`; the sample-level + `counts.total_instances`/`errored_instances` are renamed to + `counts.total_samples`/`errored_samples`; the `cases[]` array is renamed to + `tests[]`. This finishes the `case` → `test`/`sample` vocabulary migration + that ADR-0012's non-goals deferred. +- **`run_id` is a single field, not two** (`av-cpl5.2`). `summary.json`'s root + `run_id` was previously duplicated at `metadata.run_id`. The duplicate is + dropped; `run_id` lives only at the summary root going forward. Readers of + bundles written before this change should fall back to `metadata.run_id` + when the root field is absent — `summaryRunId()` in `run-artifacts.ts` + implements exactly that fallback order. +- **`index.jsonl` rows no longer inline `target_execution` or + `transcript_summary`** (`av-cpl5.3`), reversing decision point 3 above. Both + objects duplicated data already available from sidecars + (`target-execution.json` via `target_execution_path`, and each sample's + `result.json`) on every row and every repeat-sample rollup entry. Rows now + carry a compact `target_error_kind` scalar (row-level and per-`samples[]` + entry) so the target-error-kind table affordance still doesn't need a + sidecar read to know a run failed; full detail requires reading the sidecar. + Bundles written before this change still have the full objects inlined on + disk, since run bundles are immutable once written — readers that need to + support both eras can check for `target_error_kind` first and fall back to + reading the legacy `target_execution`/`transcript_summary` fields directly + off the row when present. +- **`projection_identity` is the deliberate exception and stays inline.** + Every other detailed-looking row field moved to a sidecar, but + `projection_identity` does not, and this is a permanent design decision, not + a follow-up TODO. `writeArtifactsFromResults` reads + `projection_identity.id` back off previously-written rows on disk to decide + skip/update/error duplicate policy across separate `agentv eval` invocations + that append to the same run (`indexRecordReplacementKey` / + `existingRecordsByProjectionIdentity` in `run-artifacts.ts`). Moving + `projection_identity` to a sidecar would force an N-file read on every + append just to resolve duplicate policy, and the field is already compact + enough that inlining it costs nothing comparable to `target_execution` or + `transcript_summary`. + +Net effect: `summary.json` answers run-aggregate questions, `index.jsonl` +answers dashboard-ready row-manifest questions (identity, filters, status, +compact error classification, and sidecar paths — plus the one +duplicate-policy-critical `projection_identity` exception), and per-sample +sidecars answer detailed-evidence questions. See the [Result Artifact +Contract](../../apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx) +for the current field-level shape and worked examples. + ### Multi-suite runs — one run_id, categorize by suite AND tags/experiment Confirms ADR-0009 + ADR-0012 (not a new decision): - **One `` (one timestamp) per CLI invocation**, across any number of suite YAMLs — all suites' cases live under the single `/` bundle. **Never a separate timestamp/folder per suite.** `runtime_source.eval_files` records the active eval files. @@ -367,3 +426,7 @@ exploitbench (security-exploit benchmark; AgentV research `entities/exploitbench repo-acquisition fields where they modeled authored testbed setup. - `camelCase` in an AgentV-owned artifact or response is a contract bug, not a stylistic alternative. +- Tracker `av-cpl5` locks the summary/index/sidecar boundary (see "Summary/index/sidecar + boundary is locked" above): readers written against the pre-`av-cpl5.2`/`av-cpl5.3` + shape must add the documented fallbacks (`metadata.run_id`, inline + `target_execution`/`transcript_summary`) to keep reading historical bundles. diff --git a/packages/core/test/evaluation/run-artifacts-contract.test.ts b/packages/core/test/evaluation/run-artifacts-contract.test.ts new file mode 100644 index 000000000..e1acef3f4 --- /dev/null +++ b/packages/core/test/evaluation/run-artifacts-contract.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, it } from 'bun:test'; + +import { buildProjectionIdentity } from '../../src/evaluation/projection-identity.js'; +import { + buildIndexArtifactEntry, + buildResultIndexArtifact, + buildRunSummaryArtifact, +} from '../../src/evaluation/run-artifacts.js'; +import { buildTraceFromMessages } from '../../src/evaluation/trace.js'; +import type { EvaluationResult } from '../../src/evaluation/types.js'; + +/** + * Locks the summary.json / index.jsonl / sidecar boundary shipped by + * av-cpl5.2 (cases[] -> tests[], counts rename, metadata.run_id removal) and + * av-cpl5.3 (target_execution/transcript_summary moved out of index rows). + * These assertions should fail loudly if the retired attempt-era field names + * reappear as public canonical fields. + */ + +function makeResult(overrides: Partial = {}): EvaluationResult { + const result = { + timestamp: '2026-07-06T00:00:00.000Z', + testId: 'contract-case', + score: 0.9, + assertions: [{ text: 'criterion-1', passed: true }], + output: 'test answer', + target: 'test-target', + executionStatus: 'ok', + ...overrides, + } as EvaluationResult; + + return { + ...result, + trace: + result.trace ?? + buildTraceFromMessages({ + input: [], + output: result.output ? [{ role: 'assistant', content: result.output }] : [], + finalOutput: result.output, + target: result.target, + testId: result.testId, + }), + }; +} + +describe('run summary artifact contract', () => { + it('reports tests[] and test/sample-level counts, not the retired cases/instances shape', () => { + const summary = buildRunSummaryArtifact( + [ + makeResult({ testId: 'alpha', executionStatus: 'ok' }), + makeResult({ testId: 'beta', executionStatus: 'execution_error' }), + ], + 'evals/contract.eval.yaml', + undefined, + 'run-contract-1', + ); + + expect(summary.tests).toHaveLength(2); + expect(summary.counts.total_tests).toBe(2); + expect(summary.counts.passed_tests).toBe(1); + expect(summary.counts.failed_tests).toBe(1); + expect(summary.counts.total_samples).toBe(2); + expect(summary.counts.errored_samples).toBe(1); + expect(summary.run_id).toBe('run-contract-1'); + + // Retired attempt-era fields must not reappear as public canonical fields. + expect(summary).not.toHaveProperty('cases'); + expect(summary.counts).not.toHaveProperty('total_cases'); + expect(summary.counts).not.toHaveProperty('passed_cases'); + expect(summary.counts).not.toHaveProperty('failed_cases'); + expect(summary.counts).not.toHaveProperty('total_instances'); + expect(summary.counts).not.toHaveProperty('errored_instances'); + expect(summary.metadata).not.toHaveProperty('run_id'); + expect(summary.tests[0]).not.toHaveProperty('verdict'); + }); +}); + +describe('index row artifact contract', () => { + const targetExecution: EvaluationResult['targetExecution'] = { + schemaVersion: 'agentv.target_execution.v1', + status: 'error', + targetId: 'fake-cli', + providerId: 'cli:fake-cli', + providerKind: 'cli', + runtimeMode: 'host', + command: { argv: ['fake-agent', 'run'], commandLine: 'fake-agent run', cwd: '/workspace' }, + startedAt: '2026-07-06T00:00:00.000Z', + endedAt: '2026-07-06T00:00:01.000Z', + durationMs: 1000, + exitCode: null, + signal: 'SIGSEGV', + errorKind: 'signal_crash', + message: 'target crashed', + }; + + it('keeps target runtime detail in the target_execution_path sidecar with a compact target_error_kind on the row', () => { + const result = makeResult({ + testId: 'crash-case', + executionStatus: 'execution_error', + targetExecution, + }); + + const row = buildResultIndexArtifact(result); + + expect(row.target_error_kind).toBe('signal_crash'); + expect(row.target_execution_path).toBeTruthy(); + expect(row).not.toHaveProperty('target_execution'); + expect(row).not.toHaveProperty('transcript_summary'); + expect(row).not.toHaveProperty('verdict'); + + // Repeat-sample rollups follow the same slim shape as the parent row. + expect(row.samples?.[0]?.target_error_kind).toBe('signal_crash'); + expect(row.samples?.[0]).not.toHaveProperty('target_execution'); + expect(row.samples?.[0]).not.toHaveProperty('transcript_summary'); + expect(row.samples?.[0]).not.toHaveProperty('verdict'); + }); + + it('keeps projection_identity inline on the row instead of moving it to a sidecar', () => { + const result = makeResult({ testId: 'alpha' }); + const projectionIdentity = buildProjectionIdentity({ + runId: 'run-contract-1', + evalPath: 'evals/contract.eval.yaml', + testId: 'alpha', + target: 'test-target', + envelopeId: 'envelope-1', + traceId: 'trace-1', + rootSpanId: 'span-1', + projectionFormat: 'agentv/artifacts/v1', + projectionVersion: '1', + }); + + const row = buildIndexArtifactEntry(result, { + outputDir: '/tmp/agentv-run-contract', + projectionIdentity, + }); + + // projection_identity is a deliberate exception to the sidecar-everything-else + // rule: writeArtifactsFromResults reads projection_identity.id back off + // previously-written rows on disk to decide skip/update/error duplicate + // policy across separate `agentv eval` invocations appending to the same + // run, so it stays inline to avoid an N-file sidecar read on every append. + expect(row.projection_identity?.id).toBe(projectionIdentity.id); + }); +});