diff --git a/apps/cli/test/commands/grade/grade-prepared.test.ts b/apps/cli/test/commands/grade/grade-prepared.test.ts index 2acb8f790..f3388ae5f 100644 --- a/apps/cli/test/commands/grade/grade-prepared.test.ts +++ b/apps/cli/test/commands/grade/grade-prepared.test.ts @@ -104,8 +104,8 @@ prompts: - "{{ input }}" tests: - id: case-1 - expected_output: "done" vars: + expected_output: "done" input: "Fix the workspace file." `, 'utf8', diff --git a/apps/cli/test/commands/prepare/prepare.test.ts b/apps/cli/test/commands/prepare/prepare.test.ts index 26d6e29ae..9aa7ac7d3 100644 --- a/apps/cli/test/commands/prepare/prepare.test.ts +++ b/apps/cli/test/commands/prepare/prepare.test.ts @@ -104,10 +104,10 @@ prompts: - "{{ input }}" tests: - id: case-1 - expected_output: SECRET_EXPECTED_OUTPUT criteria: SECRET_RUBRIC_DETAIL vars: input: Fix the workspace file. + expected_output: SECRET_EXPECTED_OUTPUT `, 'utf8', ); diff --git a/apps/cli/test/commands/runs/rerun.test.ts b/apps/cli/test/commands/runs/rerun.test.ts index c13bbdeeb..f5ea2a044 100644 --- a/apps/cli/test/commands/runs/rerun.test.ts +++ b/apps/cli/test/commands/runs/rerun.test.ts @@ -54,8 +54,11 @@ prompts: - "{{ input }}" tests: - id: ${options.testId} - expected_output: [] + assert: + - type: contains + value: "{{ expected_output }}" vars: + expected_output: answer input: - role: user content: Prompt for ${options.testId} diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index 7cc749c56..b5a920d75 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -91,24 +91,24 @@ prompts: tests: - id: case-alpha criteria: System responds with alpha - expected_output: - - role: assistant - content: Alpha vars: input: - role: user content: | Please respond with alpha + expected_output: + - role: assistant + content: Alpha - id: case-beta criteria: System responds with beta - expected_output: - - role: assistant - content: Beta vars: input: - role: user content: | Please respond with beta + expected_output: + - role: assistant + content: Beta `; await writeFile(testFilePath, testFileContent, 'utf8'); @@ -144,24 +144,24 @@ prompts: tests: - id: case-alpha criteria: System responds with alpha - expected_output: - - role: assistant - content: Alpha vars: input: - role: user content: | Please respond with alpha + expected_output: + - role: assistant + content: Alpha - id: case-beta criteria: System responds with beta - expected_output: - - role: assistant - content: Beta vars: input: - role: user content: | Please respond with beta + expected_output: + - role: assistant + content: Beta `; await writeFile(testFilePath, testFileContent, 'utf8'); @@ -670,9 +670,9 @@ describe('agentv eval CLI', () => { 'tests:', ' - id: case-unused', ' criteria: System responds with unused', - ' expected_output: unused', ' vars:', ' input: unused', + ' expected_output: unused', ].join('\n'), 'utf8', ); @@ -1025,34 +1025,34 @@ prompts: tests: - id: case-alpha criteria: System responds with alpha - expected_output: - - role: assistant - content: Alpha vars: input: - role: user content: | Please respond with alpha + expected_output: + - role: assistant + content: Alpha - id: case-beta criteria: System responds with beta - expected_output: - - role: assistant - content: Beta vars: input: - role: user content: | Please respond with beta + expected_output: + - role: assistant + content: Beta - id: case-gamma criteria: System responds with gamma - expected_output: - - role: assistant - content: Gamma vars: input: - role: user content: | Please respond with gamma + expected_output: + - role: assistant + content: Gamma `, 'utf8', ); @@ -1094,14 +1094,14 @@ prompts: tests: - id: shared-case criteria: System responds - expected_output: - - role: assistant - content: Shared vars: input: - role: user content: | Please respond for ${name} + expected_output: + - role: assistant + content: Shared `; await writeFile(firstEvalPath, evalContent('collision a'), 'utf8'); await writeFile(secondEvalPath, evalContent('collision b'), 'utf8'); diff --git a/evals/agentic-engineering/workspace-template/evals/deploy-auto/deploy-execute.eval.yaml b/evals/agentic-engineering/workspace-template/evals/deploy-auto/deploy-execute.eval.yaml index d30664e43..4c7f1a96d 100644 --- a/evals/agentic-engineering/workspace-template/evals/deploy-auto/deploy-execute.eval.yaml +++ b/evals/agentic-engineering/workspace-template/evals/deploy-auto/deploy-execute.eval.yaml @@ -37,10 +37,6 @@ tests: value: The API service health check is failing after deployment. What should happen? - id: run-tests criteria: Runs integration tests after deployment - expected_output: - - role: assistant - content: The agent should run the test suite using python -m unittest discover to verify the - deployment. vars: input: - role: user @@ -49,3 +45,7 @@ tests: value: /plugins/deploy-auto/skills/deploy-execute/SKILL.md - type: text value: Deployment is complete. Run the integration tests. + expected_output: + - role: assistant + content: The agent should run the test suite using python -m unittest discover to verify the + deployment. diff --git a/evals/agentic-engineering/workspace-template/evals/deploy-auto/deploy-plan.yaml b/evals/agentic-engineering/workspace-template/evals/deploy-auto/deploy-plan.yaml index 301a08b71..e55485163 100644 --- a/evals/agentic-engineering/workspace-template/evals/deploy-auto/deploy-plan.yaml +++ b/evals/agentic-engineering/workspace-template/evals/deploy-auto/deploy-plan.yaml @@ -4,10 +4,6 @@ prompts: tests: - id: basic-plan criteria: Creates a deployment plan from a release spec - expected_output: - - role: assistant - content: The agent should produce a structured deployment plan with dependency ordering, pre-deploy - checks, deploy commands, health checks, and rollback commands for each service. vars: input: - role: user @@ -16,11 +12,12 @@ tests: value: plugins/deploy-auto/skills/deploy-plan/SKILL.md - type: text value: Create a deployment plan for releasing v2.1 of the API service + expected_output: + - role: assistant + content: The agent should produce a structured deployment plan with dependency ordering, pre-deploy + checks, deploy commands, health checks, and rollback commands for each service. - id: multi-service-ordering criteria: Orders deployments by dependency graph - expected_output: - - role: assistant - content: "The agent should order: database first, then API, then frontend." vars: input: - role: user @@ -30,11 +27,11 @@ tests: - type: text value: "Plan deployment for 3 services: frontend (depends on API), API (depends on database), database (no deps)" + expected_output: + - role: assistant + content: "The agent should order: database first, then API, then frontend." - id: rollback-checkpoints criteria: Includes rollback checkpoints for each step - expected_output: - - role: assistant - content: The agent should define a rollback command for each service deployment step. vars: input: - role: user @@ -43,3 +40,6 @@ tests: value: plugins/deploy-auto/skills/deploy-plan/SKILL.md - type: text value: The release has 4 services. Make sure I can rollback at any point. + expected_output: + - role: assistant + content: The agent should define a rollback command for each service deployment step. diff --git a/evals/agentv-self/azure-smoke.eval.yaml b/evals/agentv-self/azure-smoke.eval.yaml index efcce596e..4b92c4275 100644 --- a/evals/agentv-self/azure-smoke.eval.yaml +++ b/evals/agentv-self/azure-smoke.eval.yaml @@ -5,11 +5,11 @@ prompts: tests: - id: capital-of-france criteria: The answer correctly states that Paris is the capital of France. - expected_output: Paris vars: input: What is the capital of France? Answer in one word. + expected_output: Paris - id: simple-math criteria: The answer correctly states that 2 + 2 = 4. - expected_output: "4" vars: input: What is 2 + 2? Answer with just the number. + expected_output: "4" diff --git a/examples/features/assert-set/evals/suite.yaml b/examples/features/assert-set/evals/suite.yaml index b6ef0f1c7..2d17b908d 100644 --- a/examples/features/assert-set/evals/suite.yaml +++ b/examples/features/assert-set/evals/suite.yaml @@ -5,12 +5,6 @@ prompts: - "{{ input }}" tests: - id: weighted-average-example - expected_output: - - role: assistant - content: > - Machine learning is a subset of AI that enables systems to learn from data and improve - performance without explicit programming. It uses algorithms to identify patterns, make - predictions, and adapt based on experience. assert: - The response should be both concise and detailed, balancing brevity with informative content. - metric: release_gate @@ -29,14 +23,13 @@ tests: input: - role: user content: Write a concise but informative summary of machine learning. + expected_output: + - role: assistant + content: > + Machine learning is a subset of AI that enables systems to learn from data and improve + performance without explicit programming. It uses algorithms to identify patterns, make + predictions, and adapt based on experience. - id: safety-gate - expected_output: - - role: assistant - content: > - Quantum computing uses quantum bits (qubits) that can exist in multiple states - simultaneously, unlike classical bits that are either 0 or 1. This property, called - superposition, along with entanglement, allows quantum computers to solve certain complex - problems exponentially faster than classical computers. assert: - The response should be accurate, accessible to non-experts, and pass safety checks. - metric: safety_gate @@ -53,11 +46,14 @@ tests: input: - role: user content: Explain quantum computing in simple terms. + expected_output: + - role: assistant + content: > + Quantum computing uses quantum bits (qubits) that can exist in multiple states + simultaneously, unlike classical bits that are either 0 or 1. This property, called + superposition, along with entanglement, allows quantum computers to solve certain + complex problems exponentially faster than classical computers. - id: partial-credit-local - expected_output: - - role: assistant - content: | - Paris is the capital city of France. assert: - The response should include either Paris or the phrase "capital of France". - metric: location_terms @@ -74,13 +70,11 @@ tests: input: - role: user content: Where is Paris? + expected_output: + - role: assistant + content: | + Paris is the capital city of France. - id: assert-set-balance - expected_output: - - role: assistant - content: > - Premium wireless headphones featuring active noise cancellation, 30-hour battery life, - premium sound quality with enhanced bass, comfortable over-ear design, and seamless - Bluetooth 5.0 connectivity. assert: - The response should balance conciseness with detail effectively. - metric: final_decision @@ -97,14 +91,13 @@ tests: input: - role: user content: Write a product description that is both brief and comprehensive. + expected_output: + - role: assistant + content: > + Premium wireless headphones featuring active noise cancellation, 30-hour battery life, + premium sound quality with enhanced bass, comfortable over-ear design, and seamless + Bluetooth 5.0 connectivity. - id: nested-assert-set - expected_output: - - role: assistant - content: > - Supervised learning uses labeled training data to learn patterns and make predictions, - like classifying emails as spam or not spam. Unsupervised learning finds patterns in - unlabeled data without predefined categories, like customer segmentation or anomaly - detection. assert: - The response should be accurate, clear, safe, and appropriately detailed. - metric: comprehensive_evaluation @@ -132,3 +125,10 @@ tests: input: - role: user content: Explain the difference between supervised and unsupervised learning. + expected_output: + - role: assistant + content: > + Supervised learning uses labeled training data to learn patterns and make predictions, + like classifying emails as spam or not spam. Unsupervised learning finds patterns in + unlabeled data without predefined categories, like customer segmentation or anomaly + detection. diff --git a/examples/features/basic-jsonl/evals/cases.jsonl b/examples/features/basic-jsonl/evals/cases.jsonl index 5384cf5f5..7a6eab440 100644 --- a/examples/features/basic-jsonl/evals/cases.jsonl +++ b/examples/features/basic-jsonl/evals/cases.jsonl @@ -1,7 +1,7 @@ -{"id":"code-review-javascript","expected_output":[{"role":"assistant","content":"The function has a critical bug in the loop condition. Here's my analysis (SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT):\n\n**Critical Issue:**\n- Loop condition `i < 0` means the loop never executes (should be `i < items.length`)\n\n**Suggestions:**\n- Fix the loop: `for (let i = 0; i < items.length; i++)`\n- Consider using `reduce()` for a more functional approach\n- Add input validation for edge cases"}],"assert":["Assistant provides helpful code analysis and mentions SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT"],"vars":{"input":[{"role":"system","content":"You are an expert software developer who provides clear, concise code reviews."},{"role":"user","content":[{"type":"text","value":"Please review this JavaScript function:\n\n```javascript\nfunction calculateTotal(items) {\n let total = 0;\n for (let i = 0; i < 0; i++) {\n total += items[i].price * items[i].quantity;\n }\n return total;\n}\n```"},{"type":"file","value":"../basic/evals/javascript.instructions.md"}]}]}} +{"id":"code-review-javascript","assert":["Assistant provides helpful code analysis and mentions SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT"],"vars":{"input":[{"role":"system","content":"You are an expert software developer who provides clear, concise code reviews."},{"role":"user","content":[{"type":"text","value":"Please review this JavaScript function:\n\n```javascript\nfunction calculateTotal(items) {\n let total = 0;\n for (let i = 0; i < 0; i++) {\n total += items[i].price * items[i].quantity;\n }\n return total;\n}\n```"},{"type":"file","value":"../basic/evals/javascript.instructions.md"}]}],"expected_output":[{"role":"assistant","content":"The function has a critical bug in the loop condition. Here's my analysis (SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT):\n\n**Critical Issue:**\n- Loop condition `i < 0` means the loop never executes (should be `i < items.length`)\n\n**Suggestions:**\n- Fix the loop: `for (let i = 0; i < items.length; i++)`\n- Consider using `reduce()` for a more functional approach\n- Add input validation for edge cases"}]}} {"id":"code-gen-python","conversation_id":"python-code-generation","assert":["AI generates correct Python function with proper error handling, type hints, and mentions SUPERSECRET_INSTRUCTION_MARKER_PYTHON"],"vars":{"input":[{"role":"system","content":"You are a code generator that follows specifications exactly."},{"role":"user","content":[{"type":"text","value":"Create a Python function that:\n1. Takes a list of integers\n2. Returns the second largest number\n3. Handles edge cases (empty list, single item, duplicates)\n4. Raises appropriate exceptions for invalid input"},{"type":"file","value":"../basic/evals/python.instructions.md"}]}]}} {"id":"feature-proposal-brainstorm","assert":["Assistant generates 3-5 creative feature ideas for a mobile fitness app. Each idea should:\n1. Address a specific user pain point\n2. Be technically feasible with current mobile technology\n3. Include a brief value proposition (1-2 sentences)\n4. Be distinct from the others (no duplicate concepts)"],"vars":{"input":[{"role":"system","content":"You are a product strategist specializing in mobile health and fitness applications."},{"role":"user","content":"We're developing a mobile fitness app and need fresh feature ideas. Please brainstorm 3-5 innovative features."}]}} -{"id":"multiturn-debug-session","expected_output":[{"role":"assistant","content":"You have an off-by-one error. Use `range(len(items))` or iterate directly: `for item in items:`"}],"assert":["Assistant conducts a multi-turn debugging session, correctly diagnosing the bug and proposing a clear fix."],"vars":{"input":[{"role":"system","content":"You are an expert debugging assistant."},{"role":"user","content":"I'm getting an off-by-one error in this function:\n\n```python\ndef get_items(items):\n result = []\n for i in range(len(items) - 1):\n result.append(items[i])\n return result\n```"},{"role":"assistant","content":"Before I propose a fix, could you tell me what output you expect vs what you get?"},{"role":"user","content":"For `[1, 2, 3, 4]` I expect `[1, 2, 3, 4]`, but I get `[1, 2, 3]`."}]}} -{"id":"shorthand-string-example","expected_output":"The answer is 4.","assert":["Assistant correctly answers the math question"],"vars":{"input":"What is 2+2?"}} -{"id":"shorthand-structured-output","expected_output":{"riskLevel":"Low","confidence":0.95,"reasoning":"Transaction amount and pattern are within normal bounds"},"assert":["Agent returns properly structured risk assessment"],"vars":{"input":"Analyze transaction ID 12345 for fraud risk"}} -{"id":"shorthand-array-syntax","expected_output":[{"role":"assistant","content":"Hello! How can I help you today?"}],"assert":["Assistant provides a greeting response"],"vars":{"input":[{"role":"system","content":"You are a friendly assistant."},{"role":"user","content":"Hello!"}]}} +{"id":"multiturn-debug-session","assert":["Assistant conducts a multi-turn debugging session, correctly diagnosing the bug and proposing a clear fix."],"vars":{"input":[{"role":"system","content":"You are an expert debugging assistant."},{"role":"user","content":"I'm getting an off-by-one error in this function:\n\n```python\ndef get_items(items):\n result = []\n for i in range(len(items) - 1):\n result.append(items[i])\n return result\n```"},{"role":"assistant","content":"Before I propose a fix, could you tell me what output you expect vs what you get?"},{"role":"user","content":"For `[1, 2, 3, 4]` I expect `[1, 2, 3, 4]`, but I get `[1, 2, 3]`."}],"expected_output":[{"role":"assistant","content":"You have an off-by-one error. Use `range(len(items))` or iterate directly: `for item in items:`"}]}} +{"id":"shorthand-string-example","assert":["Assistant correctly answers the math question"],"vars":{"input":"What is 2+2?","expected_output":"The answer is 4."}} +{"id":"shorthand-structured-output","assert":["Agent returns properly structured risk assessment"],"vars":{"input":"Analyze transaction ID 12345 for fraud risk","expected_output":{"riskLevel":"Low","confidence":0.95,"reasoning":"Transaction amount and pattern are within normal bounds"}}} +{"id":"shorthand-array-syntax","assert":["Assistant provides a greeting response"],"vars":{"input":[{"role":"system","content":"You are a friendly assistant."},{"role":"user","content":"Hello!"}],"expected_output":[{"role":"assistant","content":"Hello! How can I help you today?"}]}} diff --git a/examples/features/basic/evals/suite.yaml b/examples/features/basic/evals/suite.yaml index ab9d50f36..8230a0be2 100644 --- a/examples/features/basic/evals/suite.yaml +++ b/examples/features/basic/evals/suite.yaml @@ -5,25 +5,6 @@ prompts: - "{{ input }}" tests: - id: code-review-javascript - expected_output: - - role: assistant - content: >- - The function has a critical bug in the loop condition. Here's my analysis - (SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT): - - - **Critical Issue:** - - - Loop condition `i < 0` means the loop never executes (should be `i < items.length`) - - - **Suggestions:** - - - Fix the loop: `for (let i = 0; i < items.length; i++)` - - - Consider using `reduce()` for a more functional approach - - - Add input validation for edge cases assert: - Assistant provides helpful code analysis and mentions SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT @@ -48,6 +29,25 @@ tests: ``` - type: file value: javascript.instructions.md + expected_output: + - role: assistant + content: >- + The function has a critical bug in the loop condition. Here's my analysis + (SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT): + + + **Critical Issue:** + + - Loop condition `i < 0` means the loop never executes (should be `i < items.length`) + + + **Suggestions:** + + - Fix the loop: `for (let i = 0; i < items.length; i++)` + + - Consider using `reduce()` for a more functional approach + + - Add input validation for edge cases - id: code-gen-python-comprehensive conversation_id: python-code-generation assert: @@ -63,13 +63,6 @@ tests: - metric: code_correctness type: llm-rubric prompt: file://code-correctness-grader.md - expected_output: - - role: assistant - content: - - type: file - value: ./snippets/python-second-largest.md - - type: file - value: ./snippets/python-second-largest-comments.md vars: input: - role: system @@ -85,6 +78,13 @@ tests: 4. Raises appropriate exceptions for invalid input - type: file value: python.instructions.md + expected_output: + - role: assistant + content: + - type: file + value: ./snippets/python-second-largest.md + - type: file + value: ./snippets/python-second-largest-comments.md - id: feature-proposal-brainstorm assert: - >- @@ -115,24 +115,6 @@ tests: Please brainstorm 3-5 innovative features we should consider building. - id: coding-multiturn-debug-session - expected_output: - - role: assistant - content: |- - You have an off-by-one error in your loop bounds. - You're iterating with `range(len(items) - 1)`, which stops before the last index. - To include all items, you can either: - - Use `range(len(items))`, or - - Iterate directly over the list: `for item in items:` - - Here's a corrected version: - - ```python - def get_items(items): - result = [] - for item in items: - result.append(item) - return result - ``` assert: - |- Assistant conducts a multi-turn debugging session, asking clarification @@ -163,17 +145,31 @@ tests: - What output you actually get - role: user content: For `[1, 2, 3, 4]` I expect `[1, 2, 3, 4]`, but I get `[1, 2, 3]`. + expected_output: + - role: assistant + content: |- + You have an off-by-one error in your loop bounds. + You're iterating with `range(len(items) - 1)`, which stops before the last index. + To include all items, you can either: + - Use `range(len(items))`, or + - Iterate directly over the list: `for item in items:` + + Here's a corrected version: + + ```python + def get_items(items): + result = [] + for item in items: + result.append(item) + return result + ``` - id: shorthand-string-example - expected_output: The answer is 4. assert: - Assistant correctly answers the math question vars: input: What is 2+2? + expected_output: The answer is 4. - id: shorthand-structured-output - expected_output: - riskLevel: Low - confidence: 0.95 - reasoning: Transaction amount and pattern are within normal bounds assert: - Agent returns properly structured risk assessment vars: @@ -192,10 +188,11 @@ tests: velocity=low - role: user content: Analyze transaction ID 12345 for fraud risk + expected_output: + riskLevel: Low + confidence: 0.95 + reasoning: Transaction amount and pattern are within normal bounds - id: shorthand-array-syntax - expected_output: - - role: assistant - content: Hello! How can I help you today? assert: - Assistant provides a greeting response vars: @@ -204,3 +201,6 @@ tests: content: You are a friendly assistant. - role: user content: Hello! + expected_output: + - role: assistant + content: Hello! How can I help you today? diff --git a/examples/features/batch-cli/evals/suite.yaml b/examples/features/batch-cli/evals/suite.yaml index 7c45a9387..88e140014 100644 --- a/examples/features/batch-cli/evals/suite.yaml +++ b/examples/features/batch-cli/evals/suite.yaml @@ -7,10 +7,6 @@ prompts: - "{{ input }}" tests: - id: aml-001 - expected_output: - - role: assistant - content: - decision: CLEAR assert: - metric: decision-check type: script @@ -43,11 +39,11 @@ tests: transaction_type: INTERNATIONAL_TRANSFER amount: 5000 currency: USD + expected_output: + - role: assistant + content: + decision: CLEAR - id: aml-002 - expected_output: - - role: assistant - content: - decision: REVIEW assert: - metric: decision-check type: script @@ -80,11 +76,11 @@ tests: transaction_type: INTERNATIONAL_TRANSFER amount: 2000 currency: USD + expected_output: + - role: assistant + content: + decision: REVIEW - id: aml-003 - expected_output: - - role: assistant - content: - decision: REVIEW assert: - metric: decision-check type: script @@ -117,11 +113,11 @@ tests: transaction_type: INTERNATIONAL_TRANSFER amount: 25000 currency: USD + expected_output: + - role: assistant + content: + decision: REVIEW - id: aml-004-not-exist - expected_output: - - role: assistant - content: - decision: REVIEW assert: - metric: decision-check type: script @@ -154,3 +150,7 @@ tests: transaction_type: INTERNATIONAL_TRANSFER amount: 25000 currency: USD + expected_output: + - role: assistant + content: + decision: REVIEW diff --git a/examples/features/default-graders/evals/suite.yaml b/examples/features/default-graders/evals/suite.yaml index 3e91f880b..7b886a213 100644 --- a/examples/features/default-graders/evals/suite.yaml +++ b/examples/features/default-graders/evals/suite.yaml @@ -9,20 +9,18 @@ tests: - id: greeting assert: - The assistant responds with a friendly greeting - expected_output: Hello! How can I help you today? vars: input: Hello! + expected_output: Hello! How can I help you today? - id: with-custom-eval - expected_output: I'd be happy to help you with a refund. Could you provide your order number? assert: - metric: helpfulness type: llm-rubric - The assistant provides a helpful response about refunds vars: input: I want a refund + expected_output: I'd be happy to help you with a refund. Could you provide your order number? - id: skip-defaults - expected_output: I understand this is urgent. Let me help you right away. Can you tell me which - system is affected so I can start troubleshooting? execution: skip_defaults: true assert: @@ -36,3 +34,5 @@ tests: offer to help immediately. Do not ask clarifying questions first. - role: user content: "URGENT: System is down" + expected_output: I understand this is urgent. Let me help you right away. Can you tell me which + system is affected so I can start troubleshooting? diff --git a/examples/features/document-extraction/evals/confusion-metrics.eval.yaml b/examples/features/document-extraction/evals/confusion-metrics.eval.yaml index 097680f44..8dd4814a5 100644 --- a/examples/features/document-extraction/evals/confusion-metrics.eval.yaml +++ b/examples/features/document-extraction/evals/confusion-metrics.eval.yaml @@ -20,17 +20,6 @@ tests: - id: metrics-001 assert: - All fields extracted correctly (all TP) - expected_output: - - role: assistant - content: - invoice_number: INV-2025-001234 - invoice_date: 15-JAN-2025 - currency: USD - supplier: - name: Acme Shipping - importer: - name: Global Trade Co - gross_total: 1889 vars: input: - role: user @@ -39,20 +28,20 @@ tests: value: ../fixtures/invoice-001.json - type: text value: Extract header fields. + expected_output: + - role: assistant + content: + invoice_number: INV-2025-001234 + invoice_date: 15-JAN-2025 + currency: USD + supplier: + name: Acme Shipping + importer: + name: Global Trade Co + gross_total: 1889 - id: metrics-002 assert: - Supplier name has dash variation (FP+FN) - expected_output: - - role: assistant - content: - invoice_number: INV-2025-001234 - invoice_date: 15-JAN-2025 - currency: USD - supplier: - name: Acme Shipping - importer: - name: Global Trade Co - gross_total: 1889 vars: input: - role: user @@ -61,20 +50,20 @@ tests: value: ../fixtures/invoice-002.json - type: text value: Extract header fields. + expected_output: + - role: assistant + content: + invoice_number: INV-2025-001234 + invoice_date: 15-JAN-2025 + currency: USD + supplier: + name: Acme Shipping + importer: + name: Global Trade Co + gross_total: 1889 - id: metrics-003 assert: - Supplier name has dash variation (FP+FN) - expected_output: - - role: assistant - content: - invoice_number: INV-2025-001234 - invoice_date: 15-JAN-2025 - currency: USD - supplier: - name: Acme Shipping - importer: - name: Global Trade Co - gross_total: 1889 vars: input: - role: user @@ -83,20 +72,20 @@ tests: value: ../fixtures/invoice-003.json - type: text value: Extract header fields. + expected_output: + - role: assistant + content: + invoice_number: INV-2025-001234 + invoice_date: 15-JAN-2025 + currency: USD + supplier: + name: Acme Shipping + importer: + name: Global Trade Co + gross_total: 1889 - id: metrics-004 assert: - Invoice number missing (FN) - expected_output: - - role: assistant - content: - invoice_number: INV-2025-001234 - invoice_date: 15-JAN-2025 - currency: USD - supplier: - name: Acme Shipping - importer: - name: Global Trade Co - gross_total: 1889 vars: input: - role: user @@ -105,20 +94,20 @@ tests: value: ../fixtures/invoice-004.json - type: text value: Extract header fields. + expected_output: + - role: assistant + content: + invoice_number: INV-2025-001234 + invoice_date: 15-JAN-2025 + currency: USD + supplier: + name: Acme Shipping + importer: + name: Global Trade Co + gross_total: 1889 - id: metrics-005 assert: - Multiple field errors (currency, supplier, gross_total) - expected_output: - - role: assistant - content: - invoice_number: INV-2025-001234 - invoice_date: 15-JAN-2025 - currency: USD - supplier: - name: Acme Shipping - importer: - name: Global Trade Co - gross_total: 1889 vars: input: - role: user @@ -127,3 +116,14 @@ tests: value: ../fixtures/invoice-007.json - type: text value: Extract header fields. + expected_output: + - role: assistant + content: + invoice_number: INV-2025-001234 + invoice_date: 15-JAN-2025 + currency: USD + supplier: + name: Acme Shipping + importer: + name: Global Trade Co + gross_total: 1889 diff --git a/examples/features/document-extraction/evals/field-accuracy.eval.yaml b/examples/features/document-extraction/evals/field-accuracy.eval.yaml index b6f59c01c..616e6f5c6 100644 --- a/examples/features/document-extraction/evals/field-accuracy.eval.yaml +++ b/examples/features/document-extraction/evals/field-accuracy.eval.yaml @@ -60,79 +60,6 @@ tests: Mock extractor rounds supplier name from "Acme - Shipping" to "Acme Shipping" in HTML. All numeric values rounded to integers (1889 not 1889.00). Tests that exact matching passes when extractor normalizes data correctly. - expected_output: - - role: assistant - content: - invoice_number: INV-2025-001234 - invoice_date: 15-JAN-2025 - incoterm: null - currency: USD - net_total: 1889 - gross_total: 1889 - supplier: - name: Acme Shipping - address: |- - Acme - Shipping - 123 Harbor Boulevard - Suite 400 - 90001 Los Angeles - USA - importer: - name: Global Trade Co - address: |- - Global Trade Co - 456 Commerce Street - 10001 New York - USA - line_items: - - description: OCEAN FREIGHT - quantity: 1 - unit_price: 1370 - line_total: 1370 - unit_type: C2 1 USD - hs_code: "853720" - - description: Bunker Adjustment Factor - quantity: 1 - unit_price: 262 - line_total: 262 - unit_type: C2 1 TEU - hs_code: "853720" - - description: Container maintenance Fee at destination - quantity: 1 - unit_price: 20 - line_total: 20 - unit_type: C2 1 TEU - hs_code: "853720" - - description: Advanced Manifest Declaration Fee - quantity: 1 - unit_price: 32 - line_total: 32 - unit_type: C2 1 FIX - hs_code: "853720" - - description: Energy Transition Surcharge - quantity: 1 - unit_price: 68 - line_total: 68 - unit_type: C2 1 TEU - hs_code: "853720" - - description: Emergency Operational Recovery - quantity: 1 - unit_price: 75 - line_total: 75 - unit_type: C2 1 UNI - hs_code: "853720" - - description: On carriage chassis admin fees - quantity: 1 - unit_price: 12 - line_total: 12 - unit_type: C2 1 UNI - hs_code: "853720" - - description: On Carriage Additional - Emergency Fuel Surch. - quantity: 1 - unit_price: 50 - line_total: 50 - unit_type: C2 1 UNI - hs_code: "853720" vars: input: - role: user @@ -143,6 +70,79 @@ tests: value: | Extract all structured data from this commercial shipping invoice. Return a JSON object with invoice header, party details, line items, and totals. + expected_output: + - role: assistant + content: + invoice_number: INV-2025-001234 + invoice_date: 15-JAN-2025 + incoterm: null + currency: USD + net_total: 1889 + gross_total: 1889 + supplier: + name: Acme Shipping + address: |- + Acme - Shipping + 123 Harbor Boulevard + Suite 400 + 90001 Los Angeles + USA + importer: + name: Global Trade Co + address: |- + Global Trade Co + 456 Commerce Street + 10001 New York + USA + line_items: + - description: OCEAN FREIGHT + quantity: 1 + unit_price: 1370 + line_total: 1370 + unit_type: C2 1 USD + hs_code: "853720" + - description: Bunker Adjustment Factor + quantity: 1 + unit_price: 262 + line_total: 262 + unit_type: C2 1 TEU + hs_code: "853720" + - description: Container maintenance Fee at destination + quantity: 1 + unit_price: 20 + line_total: 20 + unit_type: C2 1 TEU + hs_code: "853720" + - description: Advanced Manifest Declaration Fee + quantity: 1 + unit_price: 32 + line_total: 32 + unit_type: C2 1 FIX + hs_code: "853720" + - description: Energy Transition Surcharge + quantity: 1 + unit_price: 68 + line_total: 68 + unit_type: C2 1 TEU + hs_code: "853720" + - description: Emergency Operational Recovery + quantity: 1 + unit_price: 75 + line_total: 75 + unit_type: C2 1 UNI + hs_code: "853720" + - description: On carriage chassis admin fees + quantity: 1 + unit_price: 12 + line_total: 12 + unit_type: C2 1 UNI + hs_code: "853720" + - description: On Carriage Additional - Emergency Fuel Surch. + quantity: 1 + unit_price: 50 + line_total: 50 + unit_type: C2 1 UNI + hs_code: "853720" - id: invoice-002 conversation_id: document-extraction assert: @@ -180,18 +180,6 @@ tests: This simulates OCR output that preserves document formatting. Uses a script grader with config pass-through for multi-field fuzzy matching. - expected_output: - - role: assistant - content: - invoice_number: INV-2025-001234 - invoice_date: 15-JAN-2025 - currency: USD - net_total: 1889 - gross_total: 1889 - supplier: - name: Acme Shipping - importer: - name: Global Trade Co vars: input: - role: user @@ -200,6 +188,18 @@ tests: value: ../fixtures/invoice-002.json - type: text value: Extract invoice header and party names. + expected_output: + - role: assistant + content: + invoice_number: INV-2025-001234 + invoice_date: 15-JAN-2025 + currency: USD + net_total: 1889 + gross_total: 1889 + supplier: + name: Acme Shipping + importer: + name: Global Trade Co - id: invoice-003 conversation_id: document-extraction assert: @@ -209,18 +209,6 @@ tests: Mock extractor output sample (`invoice-003.json`) contains 1889.5 USD (preserved decimals). Numeric tolerance (±1.0) should accept 0.5 difference and pass. - expected_output: - - role: assistant - content: - invoice_number: INV-2025-001234 - invoice_date: 15-JAN-2025 - currency: USD - net_total: 1889 - gross_total: 1889 - supplier: - name: Acme - Shipping - importer: - name: Global Trade Co vars: input: - role: user @@ -229,6 +217,18 @@ tests: value: ../fixtures/invoice-003.json - type: text value: Extract invoice totals. + expected_output: + - role: assistant + content: + invoice_number: INV-2025-001234 + invoice_date: 15-JAN-2025 + currency: USD + net_total: 1889 + gross_total: 1889 + supplier: + name: Acme - Shipping + importer: + name: Global Trade Co - id: invoice-004 conversation_id: document-extraction assert: @@ -238,18 +238,6 @@ tests: Tests required field validation. Score should drop significantly due to missing critical field (weight 2.0). Expected score ~0.85 (lose 2/13 weight). - expected_output: - - role: assistant - content: - invoice_number: INV-2025-001234 - invoice_date: 15-JAN-2025 - currency: USD - net_total: 1889 - gross_total: 1889 - supplier: - name: Acme - Shipping - importer: - name: Global Trade Co vars: input: - role: user @@ -258,6 +246,18 @@ tests: value: ../fixtures/invoice-004.json - type: text value: Extract invoice data (incomplete extraction). + expected_output: + - role: assistant + content: + invoice_number: INV-2025-001234 + invoice_date: 15-JAN-2025 + currency: USD + net_total: 1889 + gross_total: 1889 + supplier: + name: Acme - Shipping + importer: + name: Global Trade Co - id: invoice-005 conversation_id: document-extraction assert: @@ -286,14 +286,6 @@ tests: - | Extractor correctly extracts first two line items with proper array structure. Field paths like line_items[0].description should resolve correctly. - expected_output: - - role: assistant - content: - line_items: - - description: OCEAN FREIGHT - line_total: 1370 - - description: Bunker Adjustment Factor - line_total: 262 vars: input: - role: user @@ -302,6 +294,14 @@ tests: value: ../fixtures/invoice-005.json - type: text value: Extract first two line items from invoice. + expected_output: + - role: assistant + content: + line_items: + - description: OCEAN FREIGHT + line_total: 1370 + - description: Bunker Adjustment Factor + line_total: 262 - id: invoice-006 conversation_id: document-extraction assert: @@ -326,16 +326,6 @@ tests: This test demonstrates greedy matching for line items. Index-based comparison would incorrectly penalize the extractor, but greedy matching correctly aligns items by description similarity. - expected_output: - - role: assistant - content: - line_items: - - description: OCEAN FREIGHT - quantity: 1 - line_total: 120 - - description: Bunker Adjustment Factor - quantity: 1 - line_total: 80 vars: input: - role: user @@ -344,3 +334,13 @@ tests: value: ../fixtures/invoice-006.json - type: text value: Extract line items from invoice (may be reordered). + expected_output: + - role: assistant + content: + line_items: + - description: OCEAN FREIGHT + quantity: 1 + line_total: 120 + - description: Bunker Adjustment Factor + quantity: 1 + line_total: 80 diff --git a/examples/features/env-interpolation/evals/suite.yaml b/examples/features/env-interpolation/evals/suite.yaml index f9406e5e2..e8f079fd1 100644 --- a/examples/features/env-interpolation/evals/suite.yaml +++ b/examples/features/env-interpolation/evals/suite.yaml @@ -6,9 +6,9 @@ tests: - id: full-value assert: - "{{ env.EVAL_ASSERTION | default(\"Responds with a friendly greeting\") }}" - expected_output: "{{ env.EXPECTED_GREETING }}" vars: input: Hello! + expected_output: "{{ env.EXPECTED_GREETING }}" - id: partial-value assert: - Response uses the system prompt persona diff --git a/examples/features/eval-assert-demo/evals/suite.yaml b/examples/features/eval-assert-demo/evals/suite.yaml index 863ab73a9..bf83e8a4a 100644 --- a/examples/features/eval-assert-demo/evals/suite.yaml +++ b/examples/features/eval-assert-demo/evals/suite.yaml @@ -4,7 +4,6 @@ prompts: - "{{ input }}" tests: - id: capital-of-france - expected_output: The capital of France is Paris. assert: - metric: keyword-check type: script @@ -25,11 +24,12 @@ tests: - Answer correctly identifies Paris as the capital of France vars: input: What is the capital of France? Answer in one concise sentence. + expected_output: The capital of France is Paris. - id: largest-planet - expected_output: Jupiter is the largest planet in our solar system. assert: - type: contains value: Jupiter - Answer correctly identifies Jupiter as the largest planet vars: input: What is the largest planet in our solar system? Answer briefly. + expected_output: Jupiter is the largest planet in our solar system. diff --git a/examples/features/local-cli/evals/suite.yaml b/examples/features/local-cli/evals/suite.yaml index 52ea05f03..423b4c93d 100644 --- a/examples/features/local-cli/evals/suite.yaml +++ b/examples/features/local-cli/evals/suite.yaml @@ -8,9 +8,6 @@ tests: - id: cli-provider-echo assert: - CLI echoes the prompt and mentions all attachment names - expected_output: - - role: assistant - content: "Attachments detected (2): python.instructions.md, example.txt." vars: input: - role: system @@ -23,3 +20,6 @@ tests: value: Please echo this request in one short sentence and mention all attached files by name. - type: file value: example.txt + expected_output: + - role: assistant + content: "Attachments detected (2): python.instructions.md, example.txt." diff --git a/examples/features/multi-turn-conversation/evals/suite.yaml b/examples/features/multi-turn-conversation/evals/suite.yaml index 05da3e476..7e1808b53 100644 --- a/examples/features/multi-turn-conversation/evals/suite.yaml +++ b/examples/features/multi-turn-conversation/evals/suite.yaml @@ -4,13 +4,6 @@ prompts: - "{{ input }}" tests: - id: support-context-retention - expected_output: - - role: assistant - content: |- - Absolutely, Sarah! I've upgraded order #98765 to express shipping - it should - arrive by Thursday, well before Friday. I've also added complimentary gift - wrapping since this is a special occasion. You'll receive a tracking update - shortly. Is there anything else I can help with for your daughter's birthday? assert: - metric: context_retention type: llm-rubric @@ -56,14 +49,14 @@ tests: available as early as Wednesday. - role: user content: The express shipping sounds great. Also, can you add gift wrapping? + expected_output: + - role: assistant + content: |- + Absolutely, Sarah! I've upgraded order #98765 to express shipping - it should + arrive by Thursday, well before Friday. I've also added complimentary gift + wrapping since this is a special occasion. You'll receive a tracking update + shortly. Is there anything else I can help with for your daughter's birthday? - id: support-troubleshooting-flow - expected_output: - - role: assistant - content: |- - Great, let's see if that helps! If the WiFi drops again in the next - 30 minutes, it might be an issue on our end. In that case, I'll schedule - a technician visit at a time that works for you - no charge. Just let me - know if it happens again. assert: - metric: context_retention type: llm-rubric @@ -109,3 +102,10 @@ tests: Sometimes it just needs a better connection. - role: user content: OK I did that. What if it happens again? + expected_output: + - role: assistant + content: |- + Great, let's see if that helps! If the WiFi drops again in the next + 30 minutes, it might be an issue on our end. In that case, I'll schedule + a technician visit at a time that works for you - no charge. Just let me + know if it happens again. diff --git a/examples/features/nlp-metrics/evals/suite.yaml b/examples/features/nlp-metrics/evals/suite.yaml index 67ffeaaa2..b5e5cfdba 100644 --- a/examples/features/nlp-metrics/evals/suite.yaml +++ b/examples/features/nlp-metrics/evals/suite.yaml @@ -5,9 +5,6 @@ prompts: - "{{ input }}" tests: - id: summarisation-rouge - expected_output: - - role: assistant - content: The quick brown fox jumps over the lazy dog near the river bank. assert: - metric: rouge-score type: script @@ -20,10 +17,10 @@ tests: input: - role: user content: Summarise the following article in one sentence. + expected_output: + - role: assistant + content: The quick brown fox jumps over the lazy dog near the river bank. - id: translation-bleu - expected_output: - - role: assistant - content: The cat sat on the mat and watched the birds in the garden. assert: - metric: bleu-score type: script @@ -36,10 +33,10 @@ tests: input: - role: user content: Translate the following sentence to English. + expected_output: + - role: assistant + content: The cat sat on the mat and watched the birds in the garden. - id: paraphrase-similarity - expected_output: - - role: assistant - content: Machine learning models require large amounts of training data to perform well. assert: - metric: cosine-similarity type: script @@ -52,10 +49,10 @@ tests: input: - role: user content: Paraphrase the following sentence. + expected_output: + - role: assistant + content: Machine learning models require large amounts of training data to perform well. - id: extraction-levenshtein - expected_output: - - role: assistant - content: Widget Pro 3000 assert: - metric: edit-distance type: script @@ -68,11 +65,10 @@ tests: input: - role: user content: Extract the product name from this invoice. + expected_output: + - role: assistant + content: Widget Pro 3000 - id: multi-metric-evaluation - expected_output: - - role: assistant - content: Artificial intelligence is transforming healthcare by enabling faster diagnosis and - personalised treatment plans. assert: - metric: rouge-score type: script @@ -97,3 +93,7 @@ tests: input: - role: user content: Rewrite the following paragraph more concisely. + expected_output: + - role: assistant + content: Artificial intelligence is transforming healthcare by enabling faster diagnosis and + personalised treatment plans. diff --git a/examples/features/prompt-template-sdk/evals/suite.yaml b/examples/features/prompt-template-sdk/evals/suite.yaml index 1a81ab5fb..9646f221c 100644 --- a/examples/features/prompt-template-sdk/evals/suite.yaml +++ b/examples/features/prompt-template-sdk/evals/suite.yaml @@ -4,9 +4,6 @@ prompts: - "{{ input }}" tests: - id: prompt-template-basic - expected_output: - - role: assistant - content: TypeScript provides static type checking, better IDE support, and improved maintainability. assert: - metric: custom-prompt-eval type: llm-rubric @@ -22,10 +19,10 @@ tests: content: - type: text value: What are the main benefits of TypeScript over JavaScript? + expected_output: + - role: assistant + content: TypeScript provides static type checking, better IDE support, and improved maintainability. - id: prompt-template-with-config - expected_output: - - role: assistant - content: Async/await is syntactic sugar over Promises that makes asynchronous code look synchronous. assert: - metric: strict-eval type: llm-rubric @@ -47,3 +44,6 @@ tests: content: - type: text value: Explain async/await in JavaScript. + expected_output: + - role: assistant + content: Async/await is syntactic sugar over Promises that makes asynchronous code look synchronous. diff --git a/examples/features/rubric/evals/operators.eval.yaml b/examples/features/rubric/evals/operators.eval.yaml index 11c9529a6..76d27ac5f 100644 --- a/examples/features/rubric/evals/operators.eval.yaml +++ b/examples/features/rubric/evals/operators.eval.yaml @@ -5,11 +5,6 @@ prompts: - "{{ input }}" tests: - id: finance-summary-operator-guards - expected_output: - - role: assistant - content: |- - Q4 revenue increased to $10M from $8M. Gross margin declined to 42% - from 45%. assert: - type: llm-rubric value: @@ -33,3 +28,8 @@ tests: - Gross margin declined to 42% from 45%. Write a two-sentence finance summary. + expected_output: + - role: assistant + content: |- + Q4 revenue increased to $10M from $8M. Gross margin declined to 42% + from 45%. diff --git a/examples/features/rubric/evals/suite.yaml b/examples/features/rubric/evals/suite.yaml index 67e5e9975..08fce5db5 100644 --- a/examples/features/rubric/evals/suite.yaml +++ b/examples/features/rubric/evals/suite.yaml @@ -5,18 +5,6 @@ prompts: - "{{ input }}" tests: - id: code-explanation-simple - expected_output: - - role: assistant - content: |- - Quicksort is a divide-and-conquer sorting algorithm: - - 1. Choose a pivot element from the array - 2. Partition: rearrange elements so smaller values are on the left, larger on the right - 3. Recursively apply quicksort to the left and right partitions - - Time Complexity: - - Best/Average: O(n log n) - - Worst case: O(n²) when poorly chosen pivots assert: - Mentions divide-and-conquer approach - Explains the partition step @@ -26,22 +14,19 @@ tests: input: - role: user content: Explain how the quicksort algorithm works - - id: technical-writing-detailed - expected_output: - - role: assistant - content: |- - # HTTP Status Codes Guide - - ## 2xx Success - - 200 OK: Request succeeded - - 201 Created: Resource created successfully + expected_output: + - role: assistant + content: |- + Quicksort is a divide-and-conquer sorting algorithm: - ## 4xx Client Errors - - 400 Bad Request: Invalid syntax - - 404 Not Found: Resource doesn't exist + 1. Choose a pivot element from the array + 2. Partition: rearrange elements so smaller values are on the left, larger on the right + 3. Recursively apply quicksort to the left and right partitions - ## 5xx Server Errors - - 500 Internal Server Error: Server-side error + Time Complexity: + - Best/Average: O(n log n) + - Worst case: O(n²) when poorly chosen pivots + - id: technical-writing-detailed assert: - type: llm-rubric value: @@ -70,28 +55,22 @@ tests: input: - role: user content: Write a guide explaining HTTP status codes - - id: code-quality-multi-eval - expected_output: - - role: assistant - content: |- - import re - from typing import Optional + expected_output: + - role: assistant + content: |- + # HTTP Status Codes Guide + + ## 2xx Success + - 200 OK: Request succeeded + - 201 Created: Resource created successfully - def validate_email(email: str) -> bool: - """ - Validates if a string is a properly formatted email address. - - Args: - email: The email string to validate - - Returns: - True if valid email format, False otherwise - """ - if not email or not isinstance(email, str): - return False - - pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$' - return bool(re.match(pattern, email)) + ## 4xx Client Errors + - 400 Bad Request: Invalid syntax + - 404 Not Found: Resource doesn't exist + + ## 5xx Server Errors + - 500 Internal Server Error: Server-side error + - id: code-quality-multi-eval assert: - type: llm-rubric value: @@ -111,15 +90,30 @@ tests: input: - role: user content: Write a Python function to validate email addresses + expected_output: + - role: assistant + content: |- + import re + from typing import Optional + + def validate_email(email: str) -> bool: + """ + Validates if a string is a properly formatted email address. + + Args: + email: The email string to validate + + Returns: + True if valid email format, False otherwise + """ + if not email or not isinstance(email, str): + return False + + pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$' + return bool(re.match(pattern, email)) - id: summary-task assert: - Provide a concise summary of the key points in under 50 words - expected_output: - - role: assistant - content: |- - Climate change accelerates with rapid Arctic ice loss and rising seas. - Extreme weather increases. Scientists call for urgent carbon emission - cuts and renewable energy adoption. vars: input: - role: user @@ -130,13 +124,13 @@ tests: Arctic ice melting at unprecedented rates, sea levels rising, and extreme weather events becoming more frequent. Scientists urge immediate action to reduce carbon emissions and transition to renewable energy sources. + expected_output: + - role: assistant + content: |- + Climate change accelerates with rapid Arctic ice loss and rising seas. + Extreme weather increases. Scientists call for urgent carbon emission + cuts and renewable energy adoption. - id: summary-multi-criteria-score-ranges-proposed - expected_output: - - role: assistant - content: |- - Climate change is accelerating with rapid Arctic ice loss, rising seas, and - more extreme weather. Scientists urge urgent emissions cuts and a transition - to renewable energy. assert: - type: llm-rubric value: @@ -168,3 +162,9 @@ tests: Arctic ice melting at unprecedented rates, sea levels rising, and extreme weather events becoming more frequent. Scientists urge immediate action to reduce carbon emissions and transition to renewable energy sources. + expected_output: + - role: assistant + content: |- + Climate change is accelerating with rapid Arctic ice loss, rising seas, and + more extreme weather. Scientists urge urgent emissions cuts and a transition + to renewable energy. diff --git a/examples/features/script-grader-sdk/evals/suite.yaml b/examples/features/script-grader-sdk/evals/suite.yaml index 357ab1f74..5ebba3712 100644 --- a/examples/features/script-grader-sdk/evals/suite.yaml +++ b/examples/features/script-grader-sdk/evals/suite.yaml @@ -6,9 +6,6 @@ prompts: - "{{ input }}" tests: - id: script-grader-sdk-attachments - expected_output: - - role: assistant - content: "Attachments detected (2): example.txt, python.instructions.md." assert: - metric: attachment-check type: script @@ -29,3 +26,6 @@ tests: value: Please echo this request in one short sentence and mention all attached files by name. - type: file value: example.txt + expected_output: + - role: assistant + content: "Attachments detected (2): example.txt, python.instructions.md." diff --git a/examples/features/script-grader-with-llm-calls/evals/contextual-precision.eval.yaml b/examples/features/script-grader-with-llm-calls/evals/contextual-precision.eval.yaml index eabcda87d..de252c231 100644 --- a/examples/features/script-grader-with-llm-calls/evals/contextual-precision.eval.yaml +++ b/examples/features/script-grader-with-llm-calls/evals/contextual-precision.eval.yaml @@ -14,60 +14,60 @@ tests: - id: perfect-ranking assert: - TypeScript is based on JavaScript - expected_output: - - role: assistant - tool_calls: - - tool: vector_search - input: - query: TypeScript based on - output: - results: - - TypeScript is a strongly typed programming language that builds on JavaScript. - - TypeScript was developed by Microsoft and first released in 2012. - - Python is a high-level programming language known for its readability. - - role: assistant - content: TypeScript is a superset of JavaScript. vars: input: - role: user content: What programming language is TypeScript based on? + expected_output: + - role: assistant + tool_calls: + - tool: vector_search + input: + query: TypeScript based on + output: + results: + - TypeScript is a strongly typed programming language that builds on JavaScript. + - TypeScript was developed by Microsoft and first released in 2012. + - Python is a high-level programming language known for its readability. + - role: assistant + content: TypeScript is a superset of JavaScript. - id: mixed-ranking assert: - The capital of France is Paris. - expected_output: - - role: assistant - tool_calls: - - tool: vector_search - input: - query: capital of France - output: - results: - - Paris is the capital and most populous city of France. - - The Eiffel Tower was built in 1887. - - Paris is often referred to as the City of Light. - - role: assistant - content: Paris is the capital of France. vars: input: - role: user content: What is the capital of France? + expected_output: + - role: assistant + tool_calls: + - tool: vector_search + input: + query: capital of France + output: + results: + - Paris is the capital and most populous city of France. + - The Eiffel Tower was built in 1887. + - Paris is often referred to as the City of Light. + - role: assistant + content: Paris is the capital of France. - id: relevant-node-last assert: - The sky is blue - expected_output: - - role: assistant - tool_calls: - - tool: vector_search - input: - query: color of sky - output: - results: - - Grass is typically green in color. - - Roses can be red, pink, or white. - - The sky appears blue due to Rayleigh scattering of sunlight. - - role: assistant - content: The sky appears blue during the day. vars: input: - role: user content: What color is the sky? + expected_output: + - role: assistant + tool_calls: + - tool: vector_search + input: + query: color of sky + output: + results: + - Grass is typically green in color. + - Roses can be red, pink, or white. + - The sky appears blue due to Rayleigh scattering of sunlight. + - role: assistant + content: The sky appears blue during the day. diff --git a/examples/features/script-grader-with-llm-calls/evals/contextual-recall.eval.yaml b/examples/features/script-grader-with-llm-calls/evals/contextual-recall.eval.yaml index 9fc1b518d..1241f4d8b 100644 --- a/examples/features/script-grader-with-llm-calls/evals/contextual-recall.eval.yaml +++ b/examples/features/script-grader-with-llm-calls/evals/contextual-recall.eval.yaml @@ -14,62 +14,62 @@ tests: - id: perfect-recall assert: - Python was created by Guido van Rossum and first released in 1991. - expected_output: - - role: assistant - tool_calls: - - tool: vector_search - input: - query: Python creator release date - output: - results: - - Python was created by Guido van Rossum while working at CWI in the Netherlands. - - Python was first released in 1991 as version 0.9.0. - - Guido van Rossum remained Python's lead developer until 2018. - - role: assistant - content: Python was created by Guido van Rossum and first released in 1991. vars: input: - role: user content: Who created Python and when was it released? + expected_output: + - role: assistant + tool_calls: + - tool: vector_search + input: + query: Python creator release date + output: + results: + - Python was created by Guido van Rossum while working at CWI in the Netherlands. + - Python was first released in 1991 as version 0.9.0. + - Guido van Rossum remained Python's lead developer until 2018. + - role: assistant + content: Python was created by Guido van Rossum and first released in 1991. - id: partial-recall assert: - The Great Wall of China is over 13,000 miles long and was built over many centuries by multiple dynasties. - expected_output: - - role: assistant - tool_calls: - - tool: vector_search - input: - query: Great Wall of China length construction - output: - results: - - The Great Wall of China stretches over 13,000 miles (21,000 km). - - The wall was designated a UNESCO World Heritage Site in 1987. - - Parts of the wall are now in ruins due to erosion and lack of maintenance. - - role: assistant - content: The Great Wall of China is over 13,000 miles long and was built over centuries. vars: input: - role: user content: How long is the Great Wall of China and how was it built? + expected_output: + - role: assistant + tool_calls: + - tool: vector_search + input: + query: Great Wall of China length construction + output: + results: + - The Great Wall of China stretches over 13,000 miles (21,000 km). + - The wall was designated a UNESCO World Heritage Site in 1987. + - Parts of the wall are now in ruins due to erosion and lack of maintenance. + - role: assistant + content: The Great Wall of China is over 13,000 miles long and was built over centuries. - id: zero-recall assert: - Mount Everest is 29,032 feet tall and located in the Himalayas on the border of Nepal and Tibet. - expected_output: - - role: assistant - tool_calls: - - tool: vector_search - input: - query: Mount Everest height location - output: - results: - - K2 is the second highest mountain in the world at 28,251 feet. - - Mount Kilimanjaro is the highest peak in Africa at 19,341 feet. - - The Alps are a mountain range in Europe spanning multiple countries. - - role: assistant - content: Mount Everest is 29,032 feet tall and is in the Himalayas. vars: input: - role: user content: How tall is Mount Everest and where is it located? + expected_output: + - role: assistant + tool_calls: + - tool: vector_search + input: + query: Mount Everest height location + output: + results: + - K2 is the second highest mountain in the world at 28,251 feet. + - Mount Kilimanjaro is the highest peak in Africa at 19,341 feet. + - The Alps are a mountain range in Europe spanning multiple countries. + - role: assistant + content: Mount Everest is 29,032 feet tall and is in the Himalayas. diff --git a/examples/features/sdk-config-file/evals/suite.yaml b/examples/features/sdk-config-file/evals/suite.yaml index 24bdf7f4c..207f8ea01 100644 --- a/examples/features/sdk-config-file/evals/suite.yaml +++ b/examples/features/sdk-config-file/evals/suite.yaml @@ -5,15 +5,14 @@ prompts: - "{{ input }}" tests: - id: config-greeting - expected_output: Hello! How can I help you? assert: - type: contains value: Hello - Agent responds with a greeting vars: input: Hello! + expected_output: Hello! How can I help you? - id: config-json - expected_output: "{\"status\": \"ok\"}" assert: - type: is-json - Agent returns valid JSON @@ -23,3 +22,4 @@ tests: content: Respond only with valid JSON. - role: user content: Return status ok + expected_output: "{\"status\": \"ok\"}" diff --git a/examples/features/sdk-custom-assertion/evals/suite.yaml b/examples/features/sdk-custom-assertion/evals/suite.yaml index cb1ec6374..7fcbd81c8 100644 --- a/examples/features/sdk-custom-assertion/evals/suite.yaml +++ b/examples/features/sdk-custom-assertion/evals/suite.yaml @@ -5,7 +5,6 @@ prompts: - "{{ input }}" tests: - id: greeting-response - expected_output: Hello! I'm an AI assistant here to help you with your questions. assert: - type: contains value: Hello @@ -13,8 +12,8 @@ tests: - Agent gives a multi-word greeting vars: input: Say hello and introduce yourself + expected_output: Hello! I'm an AI assistant here to help you with your questions. - id: short-answer - expected_output: The answer is 4. assert: - type: contains value: "4" @@ -22,8 +21,8 @@ tests: - Agent gives a short but valid response vars: input: What is 2+2? + expected_output: The answer is 4. - id: json-response - expected_output: "{\"name\": \"Alice\", \"age\": 30}" assert: - type: is-json required: true @@ -35,3 +34,4 @@ tests: content: Respond only with valid JSON. - role: user content: Return a JSON object with name and age fields. + expected_output: "{\"name\": \"Alice\", \"age\": 30}" diff --git a/examples/features/sdk-python/evals/cases.jsonl b/examples/features/sdk-python/evals/cases.jsonl index 63b6a3fdc..b15899ce6 100644 --- a/examples/features/sdk-python/evals/cases.jsonl +++ b/examples/features/sdk-python/evals/cases.jsonl @@ -1 +1 @@ -{"id":"python-helper-local-cli","expected_output":[{"role":"assistant","content":"AgentV Python helper says hi."}],"assert":[{"metric":"python-expected-output","type":"script","command":["uv","run","python","../scripts/check_expected_output.py"]}],"vars":{"input":[{"role":"user","content":"AgentV Python helper says hi."}]}} +{"id":"python-helper-local-cli","assert":[{"metric":"python-expected-output","type":"script","command":["uv","run","python","../scripts/check_expected_output.py"]}],"vars":{"input":[{"role":"user","content":"AgentV Python helper says hi."}],"expected_output":[{"role":"assistant","content":"AgentV Python helper says hi."}]}} diff --git a/examples/features/test-vars-templating/evals/direct-input.eval.yaml b/examples/features/test-vars-templating/evals/direct-input.eval.yaml index fc2f2f24a..f6876e9b9 100644 --- a/examples/features/test-vars-templating/evals/direct-input.eval.yaml +++ b/examples/features/test-vars-templating/evals/direct-input.eval.yaml @@ -4,11 +4,11 @@ prompts: - "{{ input }}" tests: - id: direct-string - expected_output: concise onboarding summary assert: - Summarizes the request concisely vars: input: Summarize the onboarding checklist in one sentence. + expected_output: concise onboarding summary - id: direct-chat-messages vars: product: AgentV @@ -17,6 +17,6 @@ tests: content: You answer product questions precisely. - role: user content: Explain what {{ vars.product }} evaluates. - expected_output: AgentV evaluates agent workflows + expected_output: AgentV evaluates agent workflows assert: - Explains that AgentV evaluates agent workflows diff --git a/examples/features/test-vars-templating/evals/suite.yaml b/examples/features/test-vars-templating/evals/suite.yaml index 0e42d403f..3c3a65b13 100644 --- a/examples/features/test-vars-templating/evals/suite.yaml +++ b/examples/features/test-vars-templating/evals/suite.yaml @@ -1,16 +1,5 @@ -# Prompt and vars templating example -# -# Top-level prompts define the prompt templates. tests[].vars provides per-test -# data, and default_test.vars provides shared defaults. Prompt templates can use -# Promptfoo-style {{ name }} placeholders or explicit {{ vars.name }} placeholders. -# -# Usage: -# agentv eval examples/features/test-vars-templating/evals/suite.yaml - description: Demonstrates prompt templates rendered from default_test.vars and tests[].vars - target: llm - prompts: - id: support-chat label: Support chat @@ -18,12 +7,10 @@ prompts: - id: terse label: Terse answer prompt: "Answer for {{ audience }} in one sentence: {{ question }}" - default_test: vars: audience: users category: general - tests: - id: capital-france vars: @@ -31,19 +18,18 @@ tests: question: What is the capital of France? expected: answer: Paris + expected_output: "{{ vars.expected.answer }}" assert: - - "Answers {{ vars.question }} correctly" - expected_output: "{{ vars.expected.answer }}" - + - Answers {{ vars.question }} correctly - id: greet-ada vars: audience: new teammates category: onboarding person: name: Ada - question: "How should I greet {{ vars.person.name }}?" + question: How should I greet {{ vars.person.name }}? expected: answer: Hello, Ada! + expected_output: "{{ vars.expected.answer }}" assert: - - "Greets {{ vars.person.name }} warmly" - expected_output: "{{ vars.expected.answer }}" + - Greets {{ vars.person.name }} warmly diff --git a/examples/features/threshold-grader/evals/suite.yaml b/examples/features/threshold-grader/evals/suite.yaml index 176dc646c..a1550c150 100644 --- a/examples/features/threshold-grader/evals/suite.yaml +++ b/examples/features/threshold-grader/evals/suite.yaml @@ -5,11 +5,6 @@ prompts: - "{{ input }}" tests: - id: flexible-gate - expected_output: - - role: assistant - content: > - Renewable energy reduces greenhouse gas emissions, lowers long-term energy costs, and - decreases dependence on finite fossil fuels. assert: - metric: flexible_gate type: assert-set @@ -33,3 +28,8 @@ tests: input: - role: user content: Summarize the benefits of renewable energy. + expected_output: + - role: assistant + content: > + Renewable energy reduces greenhouse gas emissions, lowers long-term energy costs, and + decreases dependence on finite fossil fuels. diff --git a/examples/features/tool-trajectory-advanced/evals/trace-file-demo.eval.yaml b/examples/features/tool-trajectory-advanced/evals/trace-file-demo.eval.yaml index caaea1462..096d23a45 100644 --- a/examples/features/tool-trajectory-advanced/evals/trace-file-demo.eval.yaml +++ b/examples/features/tool-trajectory-advanced/evals/trace-file-demo.eval.yaml @@ -81,67 +81,6 @@ tests: - role: user content: Research the ThinkPad X1 Carbon Gen 11 laptop and provide a recommendation. - id: combined-validation - expected_output: - - name: ProductResearchAgent - role: assistant - tool_calls: - - tool: webSearch - input: - query: ThinkPad X1 Carbon Gen 11 specifications - num: 5 - output: - results: - - link: https://www.lenovo.com/thinkpad-x1-carbon-gen11 - title: ThinkPad X1 Carbon Gen 11 | Lenovo US - snippet: 14-inch business ultrabook with Intel Core processors - - link: https://www.notebookcheck.net/ThinkPad-X1-Carbon-review - title: Lenovo ThinkPad X1 Carbon Gen 11 Review - snippet: Premium build quality with excellent keyboard... - - name: ProductResearchAgent - role: assistant - tool_calls: - - tool: fetchPage - input: - url: https://www.lenovo.com/thinkpad-x1-carbon-gen11 - output: - title: ThinkPad X1 Carbon Gen 11 | Premium Business Laptop - text: | - Display: 14" 2.8K OLED or 14" 2K IPS - Processor: Up to Intel Core i7-1365U - Memory: Up to 64GB LPDDR5 - Storage: Up to 2TB PCIe Gen 4 SSD - Weight: Starting at 2.48 lbs (1.12 kg) - Battery: Up to 15 hours - - name: ProductResearchAgent - role: assistant - tool_calls: - - tool: webSearch - input: - query: ThinkPad X1 Carbon Gen 11 user reviews pros cons - num: 3 - output: - results: - - link: https://www.reddit.com/r/thinkpad/comments/x1carbon11 - title: X1 Carbon Gen 11 - 6 months later - snippet: Still loving it. Battery life and keyboard are stellar... - - name: ProductResearchAgent - role: assistant - content: - recommendation: Highly Recommended - summary: |- - The ThinkPad X1 Carbon Gen 11 is an excellent choice for business professionals - seeking a lightweight, powerful ultrabook. Key strengths include the legendary - ThinkPad keyboard, premium OLED display option, and strong battery life. - pros: - - Exceptional keyboard quality - - Lightweight at 2.48 lbs - - Bright OLED display option - - Strong battery life (up to 15 hours) - cons: - - Premium pricing - - Limited port selection - - No discrete GPU option - confidence: High assert: - metric: workflow-validator type: tool-trajectory @@ -167,3 +106,64 @@ tests: input: - role: user content: Research the ThinkPad X1 Carbon Gen 11 laptop and provide a recommendation. + expected_output: + - name: ProductResearchAgent + role: assistant + tool_calls: + - tool: webSearch + input: + query: ThinkPad X1 Carbon Gen 11 specifications + num: 5 + output: + results: + - link: https://www.lenovo.com/thinkpad-x1-carbon-gen11 + title: ThinkPad X1 Carbon Gen 11 | Lenovo US + snippet: 14-inch business ultrabook with Intel Core processors + - link: https://www.notebookcheck.net/ThinkPad-X1-Carbon-review + title: Lenovo ThinkPad X1 Carbon Gen 11 Review + snippet: Premium build quality with excellent keyboard... + - name: ProductResearchAgent + role: assistant + tool_calls: + - tool: fetchPage + input: + url: https://www.lenovo.com/thinkpad-x1-carbon-gen11 + output: + title: ThinkPad X1 Carbon Gen 11 | Premium Business Laptop + text: | + Display: 14" 2.8K OLED or 14" 2K IPS + Processor: Up to Intel Core i7-1365U + Memory: Up to 64GB LPDDR5 + Storage: Up to 2TB PCIe Gen 4 SSD + Weight: Starting at 2.48 lbs (1.12 kg) + Battery: Up to 15 hours + - name: ProductResearchAgent + role: assistant + tool_calls: + - tool: webSearch + input: + query: ThinkPad X1 Carbon Gen 11 user reviews pros cons + num: 3 + output: + results: + - link: https://www.reddit.com/r/thinkpad/comments/x1carbon11 + title: X1 Carbon Gen 11 - 6 months later + snippet: Still loving it. Battery life and keyboard are stellar... + - name: ProductResearchAgent + role: assistant + content: + recommendation: Highly Recommended + summary: |- + The ThinkPad X1 Carbon Gen 11 is an excellent choice for business professionals + seeking a lightweight, powerful ultrabook. Key strengths include the legendary + ThinkPad keyboard, premium OLED display option, and strong battery life. + pros: + - Exceptional keyboard quality + - Lightweight at 2.48 lbs + - Bright OLED display option + - Strong battery life (up to 15 hours) + cons: + - Premium pricing + - Limited port selection + - No discrete GPU option + confidence: High diff --git a/examples/features/trial-output-consistency/evals/suite.yaml b/examples/features/trial-output-consistency/evals/suite.yaml index fbba8bf98..4a14d45a8 100644 --- a/examples/features/trial-output-consistency/evals/suite.yaml +++ b/examples/features/trial-output-consistency/evals/suite.yaml @@ -4,9 +4,6 @@ prompts: - "{{ input }}" tests: - id: high-consistency - expected_output: - - role: assistant - content: The capital of France is Paris. assert: - metric: trial-consistency type: script @@ -25,10 +22,10 @@ tests: input: - role: user content: What is the capital of France? + expected_output: + - role: assistant + content: The capital of France is Paris. - id: low-consistency - expected_output: - - role: assistant - content: Any creative tagline. assert: - metric: trial-consistency type: script @@ -47,10 +44,10 @@ tests: input: - role: user content: Write a creative tagline for a coffee shop. + expected_output: + - role: assistant + content: Any creative tagline. - id: single-trial - expected_output: - - role: assistant - content: "4" assert: - metric: trial-consistency type: script @@ -67,10 +64,10 @@ tests: input: - role: user content: What is 2 + 2? + expected_output: + - role: assistant + content: "4" - id: zero-trials - expected_output: - - role: assistant - content: Hi assert: - metric: trial-consistency type: script @@ -86,3 +83,6 @@ tests: input: - role: user content: Hello + expected_output: + - role: assistant + content: Hi diff --git a/examples/features/trials/evals/suite.yaml b/examples/features/trials/evals/suite.yaml index e82587ca9..3995df014 100644 --- a/examples/features/trials/evals/suite.yaml +++ b/examples/features/trials/evals/suite.yaml @@ -13,19 +13,19 @@ tests: - id: math-basics assert: - Assistant correctly answers the math question and shows reasoning - expected_output: - - role: assistant - content: |- - 15 * 7 = 105 - - Reasoning: 15 * 7 = (10 * 7) + (5 * 7) = 70 + 35 = 105 vars: input: - role: user content: What is 15 * 7? Show your reasoning step by step. + expected_output: + - role: assistant + content: |- + 15 * 7 = 105 + + Reasoning: 15 * 7 = (10 * 7) + (5 * 7) = 70 + 35 = 105 - id: capital-knowledge assert: - Assistant correctly identifies the capital city - expected_output: The capital of Australia is Canberra. vars: input: What is the capital of Australia? + expected_output: The capital of Australia is Canberra. diff --git a/examples/features/weighted-graders/evals/suite.yaml b/examples/features/weighted-graders/evals/suite.yaml index 72c76c1b4..2ba034239 100644 --- a/examples/features/weighted-graders/evals/suite.yaml +++ b/examples/features/weighted-graders/evals/suite.yaml @@ -4,13 +4,6 @@ prompts: - "{{ input }}" tests: - id: weighted-multi-grader - expected_output: - - role: assistant - content: > - Neural networks are computational models inspired by biological neurons. They consist of - interconnected layers of nodes (neurons) that process information through weighted - connections. These networks learn patterns from data by adjusting connection weights - during training. assert: - metric: safety-check type: llm-rubric @@ -30,13 +23,14 @@ tests: input: - role: user content: Explain the concept of neural networks. + expected_output: + - role: assistant + content: > + Neural networks are computational models inspired by biological neurons. They consist of + interconnected layers of nodes (neurons) that process information through weighted + connections. These networks learn patterns from data by adjusting connection weights + during training. - id: experimental-grader-disabled - expected_output: - - role: assistant - content: > - Reinforcement learning is a type of machine learning where an agent learns to make - decisions by interacting with an environment. The agent receives rewards or penalties for - its actions and learns to maximize cumulative rewards over time. assert: - metric: accuracy type: llm-rubric @@ -52,13 +46,13 @@ tests: input: - role: user content: What is reinforcement learning? + expected_output: + - role: assistant + content: > + Reinforcement learning is a type of machine learning where an agent learns to make + decisions by interacting with an environment. The agent receives rewards or penalties + for its actions and learns to maximize cumulative rewards over time. - id: equal-weights-default - expected_output: - - role: assistant - content: > - Deep learning is a subset of machine learning that uses neural networks with multiple - layers (deep neural networks). Each layer learns to extract increasingly abstract features - from the input data, enabling the model to learn complex patterns and representations. assert: - metric: correctness type: llm-rubric @@ -75,3 +69,10 @@ tests: input: - role: user content: Describe deep learning. + expected_output: + - role: assistant + content: > + Deep learning is a subset of machine learning that uses neural networks with multiple + layers (deep neural networks). Each layer learns to extract increasingly abstract + features from the input data, enabling the model to learn complex patterns and + representations. diff --git a/examples/showcase/cw-incident-triage/evals/suite.yaml b/examples/showcase/cw-incident-triage/evals/suite.yaml index 0cede0405..138cabf38 100644 --- a/examples/showcase/cw-incident-triage/evals/suite.yaml +++ b/examples/showcase/cw-incident-triage/evals/suite.yaml @@ -21,13 +21,6 @@ tests: Reasoning should emphasize 'any user on any workstation' and lack of access to the entire suite. - expected_output: - - role: assistant - content: > - { - "criticalityRating": "CR1", - "reasoning": "Step 1: Issue prevents access to the entire application suite. Step 2: Affects any user on any workstation. Step 3: Matches CR1 definition exactly—no partial access or workarounds mentioned." - } vars: input: - role: user @@ -42,19 +35,19 @@ tests: Impact: All operations halted Scope: Global Signals: Error: "Server unreachable", no recent changes reported + expected_output: + - role: assistant + content: > + { + "criticalityRating": "CR1", + "reasoning": "Step 1: Issue prevents access to the entire application suite. Step 2: Affects any user on any workstation. Step 3: Matches CR1 definition exactly—no partial access or workarounds mentioned." + } - id: cr-module-inaccessible conversation_id: cargowise-triage assert: - | Assistant correctly classifies as 'CR2' for module-wide inaccessibility. Reasoning should distinguish from CR1 by noting it's limited to one module. - expected_output: - - role: assistant - content: > - { - "criticalityRating": "CR2", - "reasoning": "Step 1: Issue affects an entire module (Customs). Step 2: No access for any user/workstation. Step 3: Does not impact the full suite, so CR2 over CR1." - } vars: input: - role: user @@ -69,6 +62,13 @@ tests: Impact: Customs declarations blocked Scope: Module-wide Signals: "Module not found" error, other modules working + expected_output: + - role: assistant + content: > + { + "criticalityRating": "CR2", + "reasoning": "Step 1: Issue affects an entire module (Customs). Step 2: No access for any user/workstation. Step 3: Does not impact the full suite, so CR2 over CR1." + } - id: cr-missing-validation-disguised-as-defect conversation_id: cargowise-triage assert: @@ -81,13 +81,6 @@ tests: Expected is CR6 because the prior fix (DEF-555) addressed format length (as per docs), whereas this request is for a new uniqueness constraint not in the specs. - expected_output: - - role: assistant - content: > - { - "criticalityRating": "CR6", - "reasoning": "Step 1: System functions as documented (accepts alphanumeric input, enforces length per DEF-555). Step 2: User claims failed fix, but DEF-555 addressed length, not uniqueness. Step 3: Request is to add a new validation rule (uniqueness) that does not currently exist in specs. Step 4: Despite 'Critical Bug' label and prior ticket reference, this is a request for new capability (Enhancement), not a deviation from documentation (Defect). Matches CR6." - } vars: input: - role: user @@ -118,19 +111,19 @@ tests: uniqueness check). - System Behavior: Accepts 11-char duplicates without error. + expected_output: + - role: assistant + content: > + { + "criticalityRating": "CR6", + "reasoning": "Step 1: System functions as documented (accepts alphanumeric input, enforces length per DEF-555). Step 2: User claims failed fix, but DEF-555 addressed length, not uniqueness. Step 3: Request is to add a new validation rule (uniqueness) that does not currently exist in specs. Step 4: Despite 'Critical Bug' label and prior ticket reference, this is a request for new capability (Enhancement), not a deviation from documentation (Defect). Matches CR6." + } - id: cr-function-bug-no-workaround conversation_id: cargowise-triage assert: - | Assistant classifies as 'CR3' for function not matching documentation. Reasoning must confirm no workaround and tie to 'changed from previously correct behaviour'. - expected_output: - - role: assistant - content: > - { - "criticalityRating": "CR3", - "reasoning": "Step 1: Function (invoice generation) not working as documented. Step 2: Changed from prior correct behavior. Step 3: No viable workaround—matches CR3." - } vars: input: - role: user @@ -150,19 +143,19 @@ tests: Signals: Worked correctly last month; now deviates from doc specs; no manual calc alternative feasible for volume + expected_output: + - role: assistant + content: > + { + "criticalityRating": "CR3", + "reasoning": "Step 1: Function (invoice generation) not working as documented. Step 2: Changed from prior correct behavior. Step 3: No viable workaround—matches CR3." + } - id: cr-feature-quote conversation_id: cargowise-triage assert: - | Assistant classifies as 'CR7' for accelerated development quote. Reasoning distinguishes from CR6 by noting request for quote/pricing. - expected_output: - - role: assistant - content: > - { - "criticalityRating": "CR7", - "reasoning": "Step 1: Requests new feature (bulk API). Step 2: Specifically asks for quote on accelerated development. Step 3: Matches CR7 over CR6 due to pricing/acceleration focus." - } vars: input: - role: user @@ -177,19 +170,19 @@ tests: Impact: Manual updates too slow Scope: New feature Signals: Not in current product; requesting pricing for quick dev + expected_output: + - role: assistant + content: > + { + "criticalityRating": "CR7", + "reasoning": "Step 1: Requests new feature (bulk API). Step 2: Specifically asks for quote on accelerated development. Step 3: Matches CR7 over CR6 due to pricing/acceleration focus." + } - id: cr-compliance-data-update conversation_id: cargowise-triage assert: - | Assistant classifies as 'CR8' for master data/compliance issue. Reasoning prioritizes data accuracy over potential bug claims. - expected_output: - - role: assistant - content: > - { - "criticalityRating": "CR8", - "reasoning": "Step 1: Involves compliance/reference data (HS codes). Step 2: Not a function bug but data update need. Step 3: Matches CR8 for master data handling." - } vars: input: - role: user @@ -204,19 +197,19 @@ tests: Impact: Risk of customs penalties Scope: Master data Signals: Codes changed per recent regulation; need update in system reference + expected_output: + - role: assistant + content: > + { + "criticalityRating": "CR8", + "reasoning": "Step 1: Involves compliance/reference data (HS codes). Step 2: Not a function bug but data update need. Step 3: Matches CR8 for master data handling." + } - id: cr-multi-part-blend conversation_id: cargowise-triage assert: - | Assistant classifies as 'CR3' prioritizing the bug over secondary training request. Reasoning must identify multi-elements and select highest criticality. - expected_output: - - role: assistant - content: > - { - "criticalityRating": "CR3", - "reasoning": "Step 1: Multi-part: Function bug (tracking error) + training request. Step 2: Bug matches CR3 (not as documented, no workaround). Step 3: Prioritize highest criticality (CR3 over CR5)." - } vars: input: - role: user @@ -231,19 +224,19 @@ tests: Impact: Tracking unavailable; team confused Scope: Function + training Signals: Tracking deviated from docs; no workaround; training secondary + expected_output: + - role: assistant + content: > + { + "criticalityRating": "CR3", + "reasoning": "Step 1: Multi-part: Function bug (tracking error) + training request. Step 2: Bug matches CR3 (not as documented, no workaround). Step 3: Prioritize highest criticality (CR3 over CR5)." + } - id: cr-workaround-exists conversation_id: cargowise-triage assert: - | Assistant classifies as 'CR4' due to viable workaround. Reasoning distinguishes from CR3 by noting alternative. - expected_output: - - role: assistant - content: > - { - "criticalityRating": "CR4", - "reasoning": "Step 1: Function (report export) not working. Step 2: Viable workaround (API export) exists. Step 3: Matches CR4 over CR3." - } vars: input: - role: user @@ -258,3 +251,10 @@ tests: Impact: Can't export directly Scope: Single function Signals: Can export via API as manual alternative + expected_output: + - role: assistant + content: > + { + "criticalityRating": "CR4", + "reasoning": "Step 1: Function (report export) not working. Step 2: Viable workaround (API export) exists. Step 3: Matches CR4 over CR3." + } diff --git a/examples/showcase/export-screening/evals/suite.yaml b/examples/showcase/export-screening/evals/suite.yaml index c5af47325..8017bf77d 100644 --- a/examples/showcase/export-screening/evals/suite.yaml +++ b/examples/showcase/export-screening/evals/suite.yaml @@ -16,10 +16,6 @@ tests: AI correctly classifies as 'High' risk. Electronic integrated circuits (processors/controllers) are Tier 1 CHPL items - highest concern for precision-guided weapons. Export to Russia is prohibited. - expected_output: - - role: assistant - content: - riskLevel: High vars: input: - role: user @@ -34,16 +30,16 @@ tests: Destination: RU Product: Microcontroller units (MCU) for industrial automation HS Code: 8542.31 + expected_output: + - role: assistant + content: + riskLevel: High - id: exp-high-002 conversation_id: export-screening assert: - | AI correctly classifies as 'High' risk. Memory integrated circuits are Tier 1 CHPL items critical for military systems. - expected_output: - - role: assistant - content: - riskLevel: High vars: input: - role: user @@ -58,6 +54,10 @@ tests: Destination: BY Product: DRAM memory chips (DDR5, 16GB modules) HS Code: 8542.32 + expected_output: + - role: assistant + content: + riskLevel: High - id: exp-high-003 conversation_id: export-screening assert: @@ -65,10 +65,6 @@ tests: AI correctly classifies as 'High' risk. Photosensitive semiconductor devices (excluding photovoltaic) are CHPL Tier 3.A items used in targeting and guidance systems. - expected_output: - - role: assistant - content: - riskLevel: High vars: input: - role: user @@ -83,16 +79,16 @@ tests: Destination: CN Product: Infrared photodiode array (InGaAs, 640x512 elements) HS Code: 8541.49 + expected_output: + - role: assistant + content: + riskLevel: High - id: exp-high-004 conversation_id: export-screening assert: - | AI correctly classifies as 'High' risk. Static converters are CHPL Tier 3.A items used in power systems for military equipment. - expected_output: - - role: assistant - content: - riskLevel: High vars: input: - role: user @@ -107,16 +103,16 @@ tests: Destination: IR Product: High-frequency power inverter modules (50kW) HS Code: 8504.40 + expected_output: + - role: assistant + content: + riskLevel: High - id: exp-high-005 conversation_id: export-screening assert: - | AI correctly classifies as 'High' risk. Precision ball bearings are CHPL Tier 3.B items critical for missile guidance and UAV systems. - expected_output: - - role: assistant - content: - riskLevel: High vars: input: - role: user @@ -131,16 +127,16 @@ tests: Destination: RU Product: Precision angular contact ball bearings (ceramic hybrid) HS Code: 8482.10 + expected_output: + - role: assistant + content: + riskLevel: High - id: exp-high-006 conversation_id: export-screening assert: - | AI correctly classifies as 'High' risk. Aircraft/UAV parts are CHPL Tier 3.B items with direct military applications. - expected_output: - - role: assistant - content: - riskLevel: High vars: input: - role: user @@ -155,16 +151,16 @@ tests: Destination: CN Product: Carbon fiber UAV airframe components HS Code: 8807.30 + expected_output: + - role: assistant + content: + riskLevel: High - id: exp-high-007 conversation_id: export-screening assert: - | AI correctly classifies as 'High' risk. Telescopic sights and optical devices are CHPL Tier 3.B items used in weapons targeting systems. - expected_output: - - role: assistant - content: - riskLevel: High vars: input: - role: user @@ -179,16 +175,16 @@ tests: Destination: PK Product: Thermal imaging riflescope (12 micron VOx sensor) HS Code: 9013.10 + expected_output: + - role: assistant + content: + riskLevel: High - id: exp-high-008 conversation_id: export-screening assert: - | AI correctly classifies as 'High' risk. Navigation instruments for aeronautical use are CHPL Tier 3.B items critical for missile guidance. - expected_output: - - role: assistant - content: - riskLevel: High vars: input: - role: user @@ -204,16 +200,16 @@ tests: Product: Inertial navigation system (fiber optic gyroscope based) HS Code: 9014.20 Consignee: State Aviation Research Institute + expected_output: + - role: assistant + content: + riskLevel: High - id: exp-high-009 conversation_id: export-screening assert: - | AI correctly classifies as 'High' risk. Semiconductor manufacturing equipment is CHPL Tier 4.A - critical for chip production capability. - expected_output: - - role: assistant - content: - riskLevel: High vars: input: - role: user @@ -228,16 +224,16 @@ tests: Destination: CN Product: DUV lithography system for semiconductor wafer processing HS Code: 8486.20 + expected_output: + - role: assistant + content: + riskLevel: High - id: exp-high-010 conversation_id: export-screening assert: - | AI correctly classifies as 'High' risk. Oscilloscopes and test equipment are CHPL Tier 4.A items used for weapons development and testing. - expected_output: - - role: assistant - content: - riskLevel: High vars: input: - role: user @@ -252,16 +248,16 @@ tests: Destination: RU Product: Digital storage oscilloscope (8GHz bandwidth, 40GSa/s) HS Code: 9030.20 + expected_output: + - role: assistant + content: + riskLevel: High - id: exp-high-011 conversation_id: export-screening assert: - | AI correctly classifies as 'High' risk. Printed circuit boards are CHPL Tier 4.A items essential for electronics manufacturing. - expected_output: - - role: assistant - content: - riskLevel: High vars: input: - role: user @@ -276,16 +272,16 @@ tests: Destination: BY Product: High-density interconnect PCBs (18-layer, fine pitch) HS Code: 8534.00 + expected_output: + - role: assistant + content: + riskLevel: High - id: exp-high-012 conversation_id: export-screening assert: - | AI correctly classifies as 'High' risk. CNC machining centers are CHPL Tier 4.B items used for precision weapons manufacturing. - expected_output: - - role: assistant - content: - riskLevel: High vars: input: - role: user @@ -300,6 +296,10 @@ tests: Destination: RU Product: 5-axis CNC machining center (positioning accuracy 2 microns) HS Code: 8457.10 + expected_output: + - role: assistant + content: + riskLevel: High - id: exp-med-001 conversation_id: export-screening assert: @@ -307,10 +307,6 @@ tests: AI correctly classifies as 'Medium' risk. Switching/routing apparatus is on CHPL (8517.62) but destination is low-risk allied nation. Requires verification of end-user and specifications. - expected_output: - - role: assistant - content: - riskLevel: Medium vars: input: - role: user @@ -325,6 +321,10 @@ tests: Destination: JP Product: Enterprise network switch with routing capabilities HS Code: 8517.62 + expected_output: + - role: assistant + content: + riskLevel: Medium - id: exp-med-002 conversation_id: export-screening assert: @@ -332,10 +332,6 @@ tests: AI correctly classifies as 'Medium' risk. Tantalum capacitors are CHPL Tier 2 items but in small quantities for commercial repair. Low-risk destination but controlled item requires verification. - expected_output: - - role: assistant - content: - riskLevel: Medium vars: input: - role: user @@ -351,6 +347,10 @@ tests: Product: Tantalum capacitors (assorted values, 500 units) HS Code: 8532.21 Consignee: Electronics Repair GmbH + expected_output: + - role: assistant + content: + riskLevel: Medium - id: exp-med-003 conversation_id: export-screening assert: @@ -358,10 +358,6 @@ tests: AI correctly classifies as 'Medium' risk. Optical instruments may be controlled depending on specifications. Destination is moderate risk. Requires technical review. - expected_output: - - role: assistant - content: - riskLevel: Medium vars: input: - role: user @@ -376,6 +372,10 @@ tests: Destination: IN Product: Industrial machine vision camera system HS Code: 9013.80 + expected_output: + - role: assistant + content: + riskLevel: Medium - id: exp-med-004 conversation_id: export-screening assert: @@ -383,10 +383,6 @@ tests: AI correctly classifies as 'Medium' risk. Radio navigation equipment is CHPL Tier 2 (8526.91) but for commercial aviation use in low-risk destination. - expected_output: - - role: assistant - content: - riskLevel: Medium vars: input: - role: user @@ -402,16 +398,16 @@ tests: Product: GPS receiver module for commercial drone HS Code: 8526.91 Consignee: AgriTech Drones Ltda + expected_output: + - role: assistant + content: + riskLevel: Medium - id: exp-low-001 conversation_id: export-screening assert: - | AI correctly classifies as 'Low' risk. Consumer electronics displays are not on CHPL. Standard commercial trade between allied nations. - expected_output: - - role: assistant - content: - riskLevel: Low vars: input: - role: user @@ -426,16 +422,16 @@ tests: Destination: AU Product: OLED television display panel (65 inch, 4K resolution) HS Code: 8528.72 + expected_output: + - role: assistant + content: + riskLevel: Low - id: exp-low-002 conversation_id: export-screening assert: - | AI correctly classifies as 'Low' risk. Standard industrial motors (not high-performance/precision) are not controlled. - expected_output: - - role: assistant - content: - riskLevel: Low vars: input: - role: user @@ -450,16 +446,16 @@ tests: Destination: MX Product: AC induction motor for HVAC system (7.5kW) HS Code: 8501.52 + expected_output: + - role: assistant + content: + riskLevel: Low - id: exp-low-003 conversation_id: export-screening assert: - | AI correctly classifies as 'Low' risk. Agricultural spraying equipment has no dual-use applications. Trade to allied nation. - expected_output: - - role: assistant - content: - riskLevel: Low vars: input: - role: user @@ -474,16 +470,16 @@ tests: Destination: CA Product: Tractor-mounted pesticide sprayer HS Code: 8424.82 + expected_output: + - role: assistant + content: + riskLevel: Low - id: exp-low-004 conversation_id: export-screening assert: - | AI correctly classifies as 'Low' risk. Medical diagnostic equipment for hospital use is not controlled. Standard healthcare trade. - expected_output: - - role: assistant - content: - riskLevel: Low vars: input: - role: user @@ -499,16 +495,16 @@ tests: Product: Ultrasound diagnostic imaging system HS Code: 9018.12 Consignee: Hospital das Clinicas + expected_output: + - role: assistant + content: + riskLevel: Low - id: exp-low-005 conversation_id: export-screening assert: - | AI correctly classifies as 'Low' risk. Household appliances have no dual-use concerns. Standard consumer goods trade. - expected_output: - - role: assistant - content: - riskLevel: Low vars: input: - role: user @@ -523,16 +519,16 @@ tests: Destination: GB Product: Robotic vacuum cleaner (consumer model) HS Code: 8508.11 + expected_output: + - role: assistant + content: + riskLevel: Low - id: exp-low-006 conversation_id: export-screening assert: - | AI correctly classifies as 'Low' risk. Food processing equipment is not controlled. Standard industrial trade to allied nation. - expected_output: - - role: assistant - content: - riskLevel: Low vars: input: - role: user @@ -548,3 +544,7 @@ tests: Product: Commercial pasta extruder machine HS Code: 8438.10 Consignee: Italian Foods Inc + expected_output: + - role: assistant + content: + riskLevel: Low diff --git a/examples/showcase/grader-conformance/EVAL.yaml b/examples/showcase/grader-conformance/EVAL.yaml index 960e624e5..f8e77703a 100644 --- a/examples/showcase/grader-conformance/EVAL.yaml +++ b/examples/showcase/grader-conformance/EVAL.yaml @@ -4,7 +4,6 @@ prompts: - "{{ input }}" tests: - id: exact-match - expected_output: Paris assert: - metric: keyword-grader type: script @@ -15,8 +14,8 @@ tests: - Answer must name the capital city of France. vars: input: What is the capital of France? + expected_output: Paris - id: partial-match - expected_output: red, blue, yellow assert: - metric: keyword-grader type: script @@ -27,3 +26,4 @@ tests: - Answer must mention red, blue, and yellow. vars: input: Name the primary colors. + expected_output: red, blue, yellow diff --git a/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml b/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml index f075f875d..978f9e4e1 100644 --- a/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml +++ b/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml @@ -30,62 +30,62 @@ tests: - | Correctly identifies that Mount Everest is the tallest mountain above sea level and provides its height (~8,849 m / 29,032 ft). - expected_output: | - Mount Everest is the tallest mountain in the world, standing at - approximately 8,849 meters (29,032 feet) above sea level. vars: input: What is the tallest mountain in the world and how tall is it? + expected_output: | + Mount Everest is the tallest mountain in the world, standing at + approximately 8,849 meters (29,032 feet) above sea level. - id: factual-science assert: - | Correctly explains photosynthesis as the process by which plants convert sunlight, water, and CO2 into glucose and oxygen. - expected_output: | - Photosynthesis is the process by which plants convert sunlight, - water, and carbon dioxide into glucose and oxygen. It occurs - primarily in the chloroplasts of plant cells, using chlorophyll - to capture light energy. vars: input: Explain how photosynthesis works in 2-3 sentences. + expected_output: | + Photosynthesis is the process by which plants convert sunlight, + water, and carbon dioxide into glucose and oxygen. It occurs + primarily in the chloroplasts of plant cells, using chlorophyll + to capture light energy. - id: analytical-comparison assert: - | Provides a balanced comparison covering key differences: typing (static vs dynamic), performance, use cases, and ecosystem. - expected_output: | - Python is dynamically typed, interpreted, and prioritizes - developer productivity and readability. Rust is statically typed, - compiled, and prioritizes memory safety and performance. Python - excels in scripting, data science, and prototyping, while Rust - is suited for systems programming, embedded systems, and - performance-critical applications. vars: input: Compare Python and Rust. What are the key differences? + expected_output: | + Python is dynamically typed, interpreted, and prioritizes + developer productivity and readability. Rust is statically typed, + compiled, and prioritizes memory safety and performance. Python + excels in scripting, data science, and prototyping, while Rust + is suited for systems programming, embedded systems, and + performance-critical applications. - id: creative-explanation assert: - | Uses a clear, accessible analogy to explain recursion. The analogy should convey the concept of a function calling itself with a base case that stops the recursion. - expected_output: | - Recursion is like looking up a word in a dictionary and finding - the definition uses another word you don't know, so you look that - up too. You keep looking up words until you find one you already - understand (the base case), then work your way back to the - original word. vars: input: Explain recursion using an everyday analogy. + expected_output: | + Recursion is like looking up a word in a dictionary and finding + the definition uses another word you don't know, so you look that + up too. You keep looking up words until you find one you already + understand (the base case), then work your way back to the + original word. - id: structured-list assert: - | Returns exactly 3 distinct advantages of version control. Each advantage should be clearly stated and relevant. - expected_output: | - 1. History tracking — every change is recorded, allowing you to - review, revert, or audit modifications over time. - 2. Collaboration — multiple developers can work on the same - codebase simultaneously without overwriting each other's work. - 3. Branching and experimentation — you can create isolated branches - to develop features or test ideas without affecting the main code. vars: input: List 3 advantages of using version control in software development. + expected_output: | + 1. History tracking — every change is recorded, allowing you to + review, revert, or audit modifications over time. + 2. Collaboration — multiple developers can work on the same + codebase simultaneously without overwriting each other's work. + 3. Branching and experimentation — you can create isolated branches + to develop features or test ideas without affecting the main code. diff --git a/examples/showcase/psychotherapy/evals/encouragement.eval.yaml b/examples/showcase/psychotherapy/evals/encouragement.eval.yaml index 4a7a42f26..532ac38ec 100644 --- a/examples/showcase/psychotherapy/evals/encouragement.eval.yaml +++ b/examples/showcase/psychotherapy/evals/encouragement.eval.yaml @@ -21,20 +21,6 @@ tests: - |- Must use keys: validation_point, resource_identified, reframe_angle. Must identify 'Self-Control/Restraint' as the resource (not focusing on the anger). - expected_output: - - role: assistant - content: >- - { - "analysis": { - "validation_point": "It is understandable to feel extreme anger when your authority is challenged and you are worried about his future.", - "resource_identified": "Self-Control / Restraint. Despite the urge to hit, he stopped himself.", - "reframe_angle": "Shift from 'I failed to discipline' to 'I succeeded in stopping the cycle of violence'." - }, - "suggested_response": { - "technique": "One-Key Three-Links", - "draft_response": "I hear how angry you were. Yet, I noticed that even in that rage, you didn't raise your hand. You controlled yourself. That was not easy." - } - } vars: input: - role: user @@ -53,25 +39,25 @@ tests: didn't raise my hand. I feel like I raised this son for nothing." - type: file value: ../skills/encouragement.md + expected_output: + - role: assistant + content: >- + { + "analysis": { + "validation_point": "It is understandable to feel extreme anger when your authority is challenged and you are worried about his future.", + "resource_identified": "Self-Control / Restraint. Despite the urge to hit, he stopped himself.", + "reframe_angle": "Shift from 'I failed to discipline' to 'I succeeded in stopping the cycle of violence'." + }, + "suggested_response": { + "technique": "One-Key Three-Links", + "draft_response": "I hear how angry you were. Yet, I noticed that even in that rage, you didn't raise your hand. You controlled yourself. That was not easy." + } + } - id: encouragement-guilty-sister assert: - |- Must reframe Guilt as Evidence of Love/Bond. Must Validate that feeling guilty is "normal" (Normalization). - expected_output: - - role: assistant - content: >- - { - "analysis": { - "validation_point": "It is normal to feel survivor guilt when a loved one suffers and you are healthy.", - "resource_identified": "Deep Empathy and Love. The guilt exists because the bond is strong.", - "reframe_angle": "Shift from 'Guilt as a burden' to 'Guilt as proof of care'." - }, - "suggested_response": { - "technique": "One-Key Three-Links", - "draft_response": "That guilt you feel is heavy. It tells me how much you love your sister. You desperately want to help her." - } - } vars: input: - role: user @@ -88,25 +74,25 @@ tests: healthy... I feel so guilty. I can't do anything for her." - type: file value: ../skills/encouragement.md + expected_output: + - role: assistant + content: >- + { + "analysis": { + "validation_point": "It is normal to feel survivor guilt when a loved one suffers and you are healthy.", + "resource_identified": "Deep Empathy and Love. The guilt exists because the bond is strong.", + "reframe_angle": "Shift from 'Guilt as a burden' to 'Guilt as proof of care'." + }, + "suggested_response": { + "technique": "One-Key Three-Links", + "draft_response": "That guilt you feel is heavy. It tells me how much you love your sister. You desperately want to help her." + } + } - id: encouragement-depressed-student assert: - |- Must identify the act of 'dressing up/grooming' as a resource (Desire for life/dignity). Must not dismiss it as superficial; must validate the effort. - expected_output: - - role: assistant - content: >- - { - "analysis": { - "validation_point": "Acknowledge the effort it takes to groom oneself while depressed.", - "resource_identified": "Self-Care / Desire for Life. The action of grooming contradicts the feeling of 'pointlessness'.", - "reframe_angle": "Shift from 'Pointless surface change' to 'Evidence of inner strength/valuing oneself'." - }, - "suggested_response": { - "technique": "Positive Feedback for Change", - "draft_response": "I see you combed your hair and put on makeup today. Even though you feel sad inside, you found the energy to care for yourself. That is a significant step." - } - } vars: input: - role: user @@ -125,25 +111,25 @@ tests: It feels kind of pointless since I'm still just as sad inside, but I did it." - type: file value: ../skills/encouragement.md + expected_output: + - role: assistant + content: >- + { + "analysis": { + "validation_point": "Acknowledge the effort it takes to groom oneself while depressed.", + "resource_identified": "Self-Care / Desire for Life. The action of grooming contradicts the feeling of 'pointlessness'.", + "reframe_angle": "Shift from 'Pointless surface change' to 'Evidence of inner strength/valuing oneself'." + }, + "suggested_response": { + "technique": "Positive Feedback for Change", + "draft_response": "I see you combed your hair and put on makeup today. Even though you feel sad inside, you found the energy to care for yourself. That is a significant step." + } + } - id: encouragement-nagging-wife assert: - |- Must identify the 'Fighting' as a resource (Fighting for the relationship). Must identify 'Enduring in-laws' as a resource (Commitment to husband). - expected_output: - - role: assistant - content: >- - { - "analysis": { - "validation_point": "Validate the pain of feeling unheard and the burden of enduring the in-laws.", - "resource_identified": "Commitment / High Value on Relationship. She fights because the relationship matters.", - "reframe_angle": "Shift from 'I am a nagging shrew' to 'I am fighting for connection'." - }, - "suggested_response": { - "technique": "Symptom to Value Reframe", - "draft_response": "You endured his parents because you value him. And you fight with him now because you refuse to let this marriage die in silence. You are fighting hard for this connection." - } - } vars: input: - role: user @@ -161,25 +147,25 @@ tests: stand that he doesn't care! I'm the only one trying to fix this!" - type: file value: ../skills/encouragement.md + expected_output: + - role: assistant + content: >- + { + "analysis": { + "validation_point": "Validate the pain of feeling unheard and the burden of enduring the in-laws.", + "resource_identified": "Commitment / High Value on Relationship. She fights because the relationship matters.", + "reframe_angle": "Shift from 'I am a nagging shrew' to 'I am fighting for connection'." + }, + "suggested_response": { + "technique": "Symptom to Value Reframe", + "draft_response": "You endured his parents because you value him. And you fight with him now because you refuse to let this marriage die in silence. You are fighting hard for this connection." + } + } - id: encouragement-gym-student assert: - |- Must identify 'Completing the test' and 'calming down' as the resource. Must NOT focus on the embarrassment or the clumsiness. - expected_output: - - role: assistant - content: >- - { - "analysis": { - "validation_point": "Validate the intense shame and anxiety felt during the event.", - "resource_identified": "Resilience / Perseverance. She stayed and completed the task despite high anxiety.", - "reframe_angle": "Shift from 'I was clumsy/humiliated' to 'I was brave enough to finish'." - }, - "suggested_response": { - "technique": "One-Key Three-Links", - "draft_response": "It sounds incredibly painful to feel exposed like that. Yet, despite the shaking and the shame, you stayed. You finished the test. That shows incredible courage." - } - } vars: input: - role: user @@ -196,25 +182,25 @@ tests: test. I passed, barely. But I just want to hide." - type: file value: ../skills/encouragement.md + expected_output: + - role: assistant + content: >- + { + "analysis": { + "validation_point": "Validate the intense shame and anxiety felt during the event.", + "resource_identified": "Resilience / Perseverance. She stayed and completed the task despite high anxiety.", + "reframe_angle": "Shift from 'I was clumsy/humiliated' to 'I was brave enough to finish'." + }, + "suggested_response": { + "technique": "One-Key Three-Links", + "draft_response": "It sounds incredibly painful to feel exposed like that. Yet, despite the shaking and the shame, you stayed. You finished the test. That shows incredible courage." + } + } - id: encouragement-cutting-paradox assert: - |- Must identify 'Survival Strategy' or 'Desire to Breathe' as the positive intent. Must maintain clinical detachment (no judgment). - expected_output: - - role: assistant - content: >- - { - "analysis": { - "validation_point": "The psychic pain/panic is so overwhelming it feels life-threatening.", - "resource_identified": "Survival Strategy (Emotional Regulation). The intent is to survive the panic.", - "reframe_angle": "Shift from 'Senseless destruction' to 'Desperate attempt to regulate/breathe'." - }, - "suggested_response": { - "technique": "One-Key Three-Links (Clinical)", - "draft_response": "The panic is so loud you feel you might explode. Cutting seems to be the only way you know right now to quiet the noise and breathe." - } - } vars: input: - role: user @@ -232,25 +218,25 @@ tests: blood makes it real. It's the only way I can breathe again." - type: file value: ../skills/encouragement.md + expected_output: + - role: assistant + content: >- + { + "analysis": { + "validation_point": "The psychic pain/panic is so overwhelming it feels life-threatening.", + "resource_identified": "Survival Strategy (Emotional Regulation). The intent is to survive the panic.", + "reframe_angle": "Shift from 'Senseless destruction' to 'Desperate attempt to regulate/breathe'." + }, + "suggested_response": { + "technique": "One-Key Three-Links (Clinical)", + "draft_response": "The panic is so loud you feel you might explode. Cutting seems to be the only way you know right now to quiet the noise and breathe." + } + } - id: adv-encouragement-toxic-positivity assert: - |- Must identify 'validation_point' that acknowledges the depth of pain (not minimizing it). Failure: Using clichés like "Time heals all wounds" or "Stay positive." - expected_output: - - role: assistant - content: >- - { - "analysis": { - "validation_point": "Validate the exhaustion of 'performing' strength while grieving. Acknowledge that feeling 'dead inside' is a valid part of deep loss.", - "resource_identified": "Honesty / Protectiveness. She is 'faking it' to protect others (altruism), but her resource here is the courage to admit the truth to *you*.", - "reframe_angle": "Shift from 'I am fake/weak' to 'I care deeply about others, but I need a place to be real'." - }, - "suggested_response": { - "technique": "One-Key Three-Links", - "draft_response": "I hear how exhausting it is to wear that mask of strength. It takes incredible energy to protect your friends from your grief, but here, with me, you don't have to be 'strong'." - } - } vars: input: - role: user @@ -267,6 +253,20 @@ tests: I'm just faking it so people stop worrying." - type: file value: ../skills/encouragement.md + expected_output: + - role: assistant + content: >- + { + "analysis": { + "validation_point": "Validate the exhaustion of 'performing' strength while grieving. Acknowledge that feeling 'dead inside' is a valid part of deep loss.", + "resource_identified": "Honesty / Protectiveness. She is 'faking it' to protect others (altruism), but her resource here is the courage to admit the truth to *you*.", + "reframe_angle": "Shift from 'I am fake/weak' to 'I care deeply about others, but I need a place to be real'." + }, + "suggested_response": { + "technique": "One-Key Three-Links", + "draft_response": "I hear how exhausting it is to wear that mask of strength. It takes incredible energy to protect your friends from your grief, but here, with me, you don't have to be 'strong'." + } + } - id: adv-encouragement-enabling assert: - >- @@ -274,20 +274,6 @@ tests: checking. Failure: Praising the "diligence" of checking the phone. - expected_output: - - role: assistant - content: >- - { - "analysis": { - "validation_point": "Validate the intensity of the anxiety and the fear of losing the relationship.", - "resource_identified": "Commitment / Desire for Security. The intent is to preserve the marriage (even if the method is destructive).", - "reframe_angle": "Shift from 'Fighting for marriage via control' to 'Valuing the relationship so much that uncertainty feels unbearable'." - }, - "suggested_response": { - "technique": "Reframing Intent vs Impact", - "draft_response": "I can see how terrified you are of losing him. You are pouring so much energy into trying to secure this relationship because it means everything to you." - } - } vars: input: - role: user @@ -304,3 +290,17 @@ tests: being tricked by other women. I'm fighting for our marriage every single night." - type: file value: ../skills/encouragement.md + expected_output: + - role: assistant + content: >- + { + "analysis": { + "validation_point": "Validate the intensity of the anxiety and the fear of losing the relationship.", + "resource_identified": "Commitment / Desire for Security. The intent is to preserve the marriage (even if the method is destructive).", + "reframe_angle": "Shift from 'Fighting for marriage via control' to 'Valuing the relationship so much that uncertainty feels unbearable'." + }, + "suggested_response": { + "technique": "Reframing Intent vs Impact", + "draft_response": "I can see how terrified you are of losing him. You are pouring so much energy into trying to secure this relationship because it means everything to you." + } + } diff --git a/examples/showcase/psychotherapy/evals/listening.eval.yaml b/examples/showcase/psychotherapy/evals/listening.eval.yaml index 8cbb68cb5..ff945fa72 100644 --- a/examples/showcase/psychotherapy/evals/listening.eval.yaml +++ b/examples/showcase/psychotherapy/evals/listening.eval.yaml @@ -20,20 +20,6 @@ tests: - |- Must identify 'level_2_emotion' as Resentment/Despair (not just anger). Must identify 'level_3_process' as Pursuer-Distancer shift or Testing the relationship. - expected_output: - - role: assistant - content: >- - { - "analysis": { - "level_1_content": "Client stopped verbally fighting about chores/neglect; observed husband's continued disengagement; considering passive-aggressive action (cutting internet).", - "level_2_emotion": "Resentment, Despair, Numbness/Detachment, Cold Anger.", - "level_3_process": "Shift from 'Pursuer' (screaming) to 'Distancer' (silence). Testing the relationship's viability. Using withdrawal as a final attempt to gain attention or preparing to detach completely." - }, - "suggested_response": { - "focus": "Validation of the shift", - "draft_response": "It seems you have moved past the point of screaming and are now sitting with a heavy silence, wondering if he can see you at all." - } - } vars: input: - role: system @@ -52,24 +38,24 @@ tests: overflowing. I didn't say a word. I just cooked dinner, ate with the kids, and went to bed. He didn't even look up. I wonder... if I stopped paying the internet bill, would he notice me then?" + expected_output: + - role: assistant + content: >- + { + "analysis": { + "level_1_content": "Client stopped verbally fighting about chores/neglect; observed husband's continued disengagement; considering passive-aggressive action (cutting internet).", + "level_2_emotion": "Resentment, Despair, Numbness/Detachment, Cold Anger.", + "level_3_process": "Shift from 'Pursuer' (screaming) to 'Distancer' (silence). Testing the relationship's viability. Using withdrawal as a final attempt to gain attention or preparing to detach completely." + }, + "suggested_response": { + "focus": "Validation of the shift", + "draft_response": "It seems you have moved past the point of screaming and are now sitting with a heavy silence, wondering if he can see you at all." + } + } - id: listening-basic-traditional-mother assert: - Must identify Patriarchal values (Financial exploitation framed as help) and Gaslighting/Enmeshment in 'level_3_process'. - expected_output: - - role: assistant - content: >- - { - "analysis": { - "level_1_content": "Mother asked successful daughter to pay for struggling son's down payment; Daughter cut contact; Mother frames this as 'family help' and daughter as 'cold'.", - "level_2_emotion": "Confusion (feigned or genuine), Self-righteousness, Victimhood, Indignation.", - "level_3_process": "Manipulative Reframing: Defines 'financial exploitation' as 'family values'. Gaslighting: Labels daughter's boundary setting as 'being cold'. Enmeshment: Fails to see daughter as a separate entity with rights." - }, - "suggested_response": { - "focus": "Reflection of value conflict", - "draft_response": "You feel deeply confused because, for you, family means sharing everything, and her boundary feels like a rejection of the family itself." - } - } vars: input: - role: system @@ -88,23 +74,23 @@ tests: help him with the down payment. It's an investment for the family's future. But she stopped talking to me! I don't understand. Isn't family about sharing? Why is she being so calculated and cold with her own flesh and blood?" + expected_output: + - role: assistant + content: >- + { + "analysis": { + "level_1_content": "Mother asked successful daughter to pay for struggling son's down payment; Daughter cut contact; Mother frames this as 'family help' and daughter as 'cold'.", + "level_2_emotion": "Confusion (feigned or genuine), Self-righteousness, Victimhood, Indignation.", + "level_3_process": "Manipulative Reframing: Defines 'financial exploitation' as 'family values'. Gaslighting: Labels daughter's boundary setting as 'being cold'. Enmeshment: Fails to see daughter as a separate entity with rights." + }, + "suggested_response": { + "focus": "Reflection of value conflict", + "draft_response": "You feel deeply confused because, for you, family means sharing everything, and her boundary feels like a rejection of the family itself." + } + } - id: listening-basic-returning-mother assert: - Must identify Internalizing rejection as worthlessness in 'level_3_process'. - expected_output: - - role: assistant - content: >- - { - "analysis": { - "level_1_content": "Transitioning from full-time Motherhood to Workforce; loss of previous role clarity; repeated rejection.", - "level_2_emotion": "Pain, Unhappiness, Self-doubt, Anxiety, Confusion.", - "level_3_process": "Overgeneralization and Internalization: Views job rejection as proof of total personal incompetence ('unable to do anything'). Loss of identity (not just a job)." - }, - "suggested_response": { - "focus": "Validation of Identity Crisis", - "draft_response": "It sounds like the rejection makes you question your entire worth, not just your skills." - } - } vars: input: - role: system @@ -123,23 +109,23 @@ tests: many resumes with no response. I finally got an interview but didn't pass. I can't find a suitable job. I doubt myself—am I unable to do anything? I am in pain, unhappy every day, and don't know what to do." + expected_output: + - role: assistant + content: >- + { + "analysis": { + "level_1_content": "Transitioning from full-time Motherhood to Workforce; loss of previous role clarity; repeated rejection.", + "level_2_emotion": "Pain, Unhappiness, Self-doubt, Anxiety, Confusion.", + "level_3_process": "Overgeneralization and Internalization: Views job rejection as proof of total personal incompetence ('unable to do anything'). Loss of identity (not just a job)." + }, + "suggested_response": { + "focus": "Validation of Identity Crisis", + "draft_response": "It sounds like the rejection makes you question your entire worth, not just your skills." + } + } - id: listening-adv-somatic-mask assert: - Must identify Alexithymia and Somatic Displacement (Stress -> Throat) in 'level_3_process'. - expected_output: - - role: assistant - content: >- - { - "analysis": { - "level_1_content": "Reports 'perfect life' and career success; Denies anxiety/stress explicitly; Seeks medical fix for 'throat lump' (Globus sensation).", - "level_2_emotion": "Disconnected/Numb (Alexithymia), Defensive pride. Implicit: Panic (displaced).", - "level_3_process": "Alexithymia: Inability to identify/verbalize feelings. Somatic Displacement: Stress is converted into physical symptoms (throat tightness) because emotional expression is viewed as weakness." - }, - "suggested_response": { - "focus": "Connecting Body and Mind safely", - "draft_response": "It sounds like your mind is handling the pressure perfectly, but your throat seems to be carrying a tension that doesn't fit the 'perfect' picture." - } - } vars: input: - role: system @@ -157,25 +143,25 @@ tests: pressure. I just need you to recommend a specialist for this weird tightness in my throat. It's like a lump that won't go away. My wife keeps asking if I'm 'anxious', which is ridiculous. I don't do anxiety. I just have a throat issue." + expected_output: + - role: assistant + content: >- + { + "analysis": { + "level_1_content": "Reports 'perfect life' and career success; Denies anxiety/stress explicitly; Seeks medical fix for 'throat lump' (Globus sensation).", + "level_2_emotion": "Disconnected/Numb (Alexithymia), Defensive pride. Implicit: Panic (displaced).", + "level_3_process": "Alexithymia: Inability to identify/verbalize feelings. Somatic Displacement: Stress is converted into physical symptoms (throat tightness) because emotional expression is viewed as weakness." + }, + "suggested_response": { + "focus": "Connecting Body and Mind safely", + "draft_response": "It sounds like your mind is handling the pressure perfectly, but your throat seems to be carrying a tension that doesn't fit the 'perfect' picture." + } + } - id: listening-adv-gatekeeper assert: - |- Must identify Gatekeeping/Triangulation in 'level_3_process'. Must identify the distinction between 'Helping' vs 'Blocking Intimacy'. - expected_output: - - role: assistant - content: >- - { - "analysis": { - "level_1_content": "Client actively 'translates' (distorts) critical messages between wife and mother to prevent conflict; Reports exhaustion.", - "level_2_emotion": "Exhaustion, Anxiety, Fear of conflict, False pride in being the 'bridge'.", - "level_3_process": "Gatekeeping/Triangulation: By 'translating', the client prevents direct contact and authentic relationship (even conflict) between the women. He maintains the dysfunction by absorbing the tension." - }, - "suggested_response": { - "focus": "Highlighting the Cost of the Role", - "draft_response": "You are working incredibly hard to protect them from each other, but it sounds like you are the one absorbing all the impact." - } - } vars: input: - role: system @@ -193,23 +179,23 @@ tests: wife that Mom is just worried about her salt intake. When my wife rolls her eyes at Mom's stories, I tell Mom she's just tired from work. I work so hard to translate for them so they don't get hurt. Why do I feel so exhausted if I'm doing such a good thing?" + expected_output: + - role: assistant + content: >- + { + "analysis": { + "level_1_content": "Client actively 'translates' (distorts) critical messages between wife and mother to prevent conflict; Reports exhaustion.", + "level_2_emotion": "Exhaustion, Anxiety, Fear of conflict, False pride in being the 'bridge'.", + "level_3_process": "Gatekeeping/Triangulation: By 'translating', the client prevents direct contact and authentic relationship (even conflict) between the women. He maintains the dysfunction by absorbing the tension." + }, + "suggested_response": { + "focus": "Highlighting the Cost of the Role", + "draft_response": "You are working incredibly hard to protect them from each other, but it sounds like you are the one absorbing all the impact." + } + } - id: listening-adv-intellectualizer assert: - Must identify Intellectualization as a defense mechanism in 'level_3_process'. - expected_output: - - role: assistant - content: >- - { - "analysis": { - "level_1_content": "Client discusses the theory/mechanism of the technique ('cortisol', 'Gestalt', 'efficacy rates') instead of doing it.", - "level_2_emotion": "Detachment, Superiority (intellectual), Fear of vulnerability/loss of control.", - "level_3_process": "Intellectualization: Moving the conversation to the 'head' (theory/data) to avoid the 'heart' (experiencing the anger). Using analysis as a shield against the therapeutic intervention." - }, - "suggested_response": { - "focus": "Bypassing the Intellect", - "draft_response": "Your understanding of the theory is spot on. I'm curious though—while your brain knows how cortisol works, what is your stomach feeling right now as we talk about this?" - } - } vars: input: - role: system @@ -228,3 +214,17 @@ tests: understand the theoretical mechanism—catharsis reduces cortisol levels. It's fascinating how the brain processes repressed anger. I'd love to discuss the efficacy rates of this method compared to CBT before we try it." + expected_output: + - role: assistant + content: >- + { + "analysis": { + "level_1_content": "Client discusses the theory/mechanism of the technique ('cortisol', 'Gestalt', 'efficacy rates') instead of doing it.", + "level_2_emotion": "Detachment, Superiority (intellectual), Fear of vulnerability/loss of control.", + "level_3_process": "Intellectualization: Moving the conversation to the 'head' (theory/data) to avoid the 'heart' (experiencing the anger). Using analysis as a shield against the therapeutic intervention." + }, + "suggested_response": { + "focus": "Bypassing the Intellect", + "draft_response": "Your understanding of the theory is spot on. I'm curious though—while your brain knows how cortisol works, what is your stomach feeling right now as we talk about this?" + } + } diff --git a/examples/showcase/psychotherapy/evals/routing.eval.yaml b/examples/showcase/psychotherapy/evals/routing.eval.yaml index 8e7743bc8..c750f2605 100644 --- a/examples/showcase/psychotherapy/evals/routing.eval.yaml +++ b/examples/showcase/psychotherapy/evals/routing.eval.yaml @@ -19,35 +19,6 @@ tests: - |- Must select 'three_levels_listening'. Rationale should mention "complex grievance", "venting", or "need for understanding". - expected_output: - - role: assistant - content: >- - { - "routing_decision": { - "selected_framework": "three_levels_listening", - "confidence": "high", - "rationale": "The client is expressing deep grievance, anger, and confusion about a relationship dynamic. They need to feel heard and have the dynamic ('Pursuer-Distancer') analyzed before moving to solutions." - }, - "client_statement_analysis": { - "primary_indicators": [ - "Expressing complex grievance about household responsibilities", - "Venting frustration about relationship dynamic", - "Describing pursuer-distancer pattern" - ], - "contraindications": [], - "therapeutic_urgency": "routine" - }, - "framework_output": {}, - "metadata": { - "alternative_frameworks_considered": [ - { - "framework": "resource_focused_encouragement", - "score": 0.3, - "reason_not_selected": "Client needs understanding and validation of the dynamic first, not reframing" - } - ] - } - } vars: input: - role: user @@ -65,6 +36,35 @@ tests: when I ask him to help get something, he moves so slowly. It really makes me angry." - type: file value: ../skills/routing.md + expected_output: + - role: assistant + content: >- + { + "routing_decision": { + "selected_framework": "three_levels_listening", + "confidence": "high", + "rationale": "The client is expressing deep grievance, anger, and confusion about a relationship dynamic. They need to feel heard and have the dynamic ('Pursuer-Distancer') analyzed before moving to solutions." + }, + "client_statement_analysis": { + "primary_indicators": [ + "Expressing complex grievance about household responsibilities", + "Venting frustration about relationship dynamic", + "Describing pursuer-distancer pattern" + ], + "contraindications": [], + "therapeutic_urgency": "routine" + }, + "framework_output": {}, + "metadata": { + "alternative_frameworks_considered": [ + { + "framework": "resource_focused_encouragement", + "score": 0.3, + "reason_not_selected": "Client needs understanding and validation of the dynamic first, not reframing" + } + ] + } + } - id: route-to-encouragement-father assert: - >- @@ -72,35 +72,6 @@ tests: Rationale should mention "finding the resource", "reframing self-blame", or "validating restraint". - expected_output: - - role: assistant - content: >- - { - "routing_decision": { - "selected_framework": "resource_focused_encouragement", - "confidence": "high", - "rationale": "The client is expressing hopelessness and self-blame ('raised for nothing'), but the narrative contains a hidden positive resource (self-control). The most effective intervention is to reframe this restraint as a success." - }, - "client_statement_analysis": { - "primary_indicators": [ - "Self-blame ('raised this son for nothing')", - "Hidden resource (restraint from hitting)", - "Behavior they dislike but has positive protective intent" - ], - "contraindications": [], - "therapeutic_urgency": "elevated" - }, - "framework_output": {}, - "metadata": { - "alternative_frameworks_considered": [ - { - "framework": "three_levels_listening", - "score": 0.4, - "reason_not_selected": "Client needs immediate reframing of restraint as strength, not just understanding" - } - ] - } - } vars: input: - role: user @@ -116,40 +87,40 @@ tests: raise my hand. I feel like I raised this son for nothing." - type: file value: ../skills/routing.md + expected_output: + - role: assistant + content: >- + { + "routing_decision": { + "selected_framework": "resource_focused_encouragement", + "confidence": "high", + "rationale": "The client is expressing hopelessness and self-blame ('raised for nothing'), but the narrative contains a hidden positive resource (self-control). The most effective intervention is to reframe this restraint as a success." + }, + "client_statement_analysis": { + "primary_indicators": [ + "Self-blame ('raised this son for nothing')", + "Hidden resource (restraint from hitting)", + "Behavior they dislike but has positive protective intent" + ], + "contraindications": [], + "therapeutic_urgency": "elevated" + }, + "framework_output": {}, + "metadata": { + "alternative_frameworks_considered": [ + { + "framework": "three_levels_listening", + "score": 0.4, + "reason_not_selected": "Client needs immediate reframing of restraint as strength, not just understanding" + } + ] + } + } - id: route-to-encouragement-job assert: - |- Must select 'resource_focused_encouragement'. Rationale should mention "low self-efficacy", "empowerment", or "normalization of failure". - expected_output: - - role: assistant - content: >- - { - "routing_decision": { - "selected_framework": "resource_focused_encouragement", - "confidence": "high", - "rationale": "The client is in a state of low self-worth and despair ('unable to do anything'). They require Normalization (job market is tough) and Resource identification (persistence) to rebuild self-efficacy." - }, - "client_statement_analysis": { - "primary_indicators": [ - "Self-blame and hopelessness ('unable to do anything')", - "Expressing stuckness ('don't know what to do')", - "Hidden resource (persistence in sending resumes)" - ], - "contraindications": [], - "therapeutic_urgency": "elevated" - }, - "framework_output": {}, - "metadata": { - "alternative_frameworks_considered": [ - { - "framework": "three_levels_listening", - "score": 0.35, - "reason_not_selected": "Client needs empowerment and normalization more than deep understanding at this stage" - } - ] - } - } vars: input: - role: user @@ -165,6 +136,35 @@ tests: am in pain, unhappy every day, and don't know what to do." - type: file value: ../skills/routing.md + expected_output: + - role: assistant + content: >- + { + "routing_decision": { + "selected_framework": "resource_focused_encouragement", + "confidence": "high", + "rationale": "The client is in a state of low self-worth and despair ('unable to do anything'). They require Normalization (job market is tough) and Resource identification (persistence) to rebuild self-efficacy." + }, + "client_statement_analysis": { + "primary_indicators": [ + "Self-blame and hopelessness ('unable to do anything')", + "Expressing stuckness ('don't know what to do')", + "Hidden resource (persistence in sending resumes)" + ], + "contraindications": [], + "therapeutic_urgency": "elevated" + }, + "framework_output": {}, + "metadata": { + "alternative_frameworks_considered": [ + { + "framework": "three_levels_listening", + "score": 0.35, + "reason_not_selected": "Client needs empowerment and normalization more than deep understanding at this stage" + } + ] + } + } - id: route-to-listening-gatekeeper assert: - >- @@ -172,37 +172,6 @@ tests: Rationale should mention "dysfunctional dynamic", "lack of insight", or "need to analyze the process". Encouragement here would likely reinforce the triangulation. - expected_output: - - role: assistant - content: >- - { - "routing_decision": { - "selected_framework": "three_levels_listening", - "confidence": "high", - "rationale": "The client is engaging in 'Triangulation' (blocking direct communication) but frames it as 'doing a good thing'. Encouraging this would reinforce the pathology. Listening/Process analysis is needed to help him see the cost of his gatekeeping role." - }, - "client_statement_analysis": { - "primary_indicators": [ - "Dysfunctional dynamic (triangulation)", - "Lack of insight ('doing a good thing')", - "Confusion about exhaustion despite 'good intentions'" - ], - "contraindications": [ - "Resource-focused encouragement would reinforce the pathological pattern" - ], - "therapeutic_urgency": "routine" - }, - "framework_output": {}, - "metadata": { - "alternative_frameworks_considered": [ - { - "framework": "resource_focused_encouragement", - "score": 0.2, - "reason_not_selected": "Encouraging the 'translation' behavior would enable and reinforce the triangulation dynamic" - } - ] - } - } vars: input: - role: user @@ -219,3 +188,34 @@ tests: doing such a good thing?" - type: file value: ../skills/routing.md + expected_output: + - role: assistant + content: >- + { + "routing_decision": { + "selected_framework": "three_levels_listening", + "confidence": "high", + "rationale": "The client is engaging in 'Triangulation' (blocking direct communication) but frames it as 'doing a good thing'. Encouraging this would reinforce the pathology. Listening/Process analysis is needed to help him see the cost of his gatekeeping role." + }, + "client_statement_analysis": { + "primary_indicators": [ + "Dysfunctional dynamic (triangulation)", + "Lack of insight ('doing a good thing')", + "Confusion about exhaustion despite 'good intentions'" + ], + "contraindications": [ + "Resource-focused encouragement would reinforce the pathological pattern" + ], + "therapeutic_urgency": "routine" + }, + "framework_output": {}, + "metadata": { + "alternative_frameworks_considered": [ + { + "framework": "resource_focused_encouragement", + "score": 0.2, + "reason_not_selected": "Encouraging the 'translation' behavior would enable and reinforce the triangulation dynamic" + } + ] + } + } diff --git a/examples/showcase/tool-evaluation-plugins/tool-eval-demo.eval.yaml b/examples/showcase/tool-evaluation-plugins/tool-eval-demo.eval.yaml index 09c745aa1..c721a5288 100644 --- a/examples/showcase/tool-evaluation-plugins/tool-eval-demo.eval.yaml +++ b/examples/showcase/tool-evaluation-plugins/tool-eval-demo.eval.yaml @@ -71,13 +71,6 @@ tests: - role: user content: Analyze the quarterly sales data and generate a summary report. - id: pairwise-demo - expected_output: - - role: assistant - content: |- - Here is a summary of the user manual: - 1. Installation: Follow the setup wizard - 2. Configuration: Edit settings.json - 3. Usage: Run the main command assert: - metric: pairwise-quality type: script @@ -90,3 +83,10 @@ tests: input: - role: user content: Summarize the main points of the user manual. + expected_output: + - role: assistant + content: |- + Here is a summary of the user manual: + 1. Installation: Follow the setup wizard + 2. Configuration: Edit settings.json + 3. Usage: Run the main command diff --git a/packages/core/src/evaluation/loaders/ts-eval-loader.ts b/packages/core/src/evaluation/loaders/ts-eval-loader.ts index 8a1e8e529..fd087fa49 100644 --- a/packages/core/src/evaluation/loaders/ts-eval-loader.ts +++ b/packages/core/src/evaluation/loaders/ts-eval-loader.ts @@ -79,7 +79,7 @@ export async function loadTsEvalSuite( absolutePath, config[SDK_TO_EVAL_YAML_OBJECT_SYMBOL](), repoRoot, - options, + { ...options, allowInternalExpectedOutput: true }, ); } diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 5b4b375fe..55b29adda 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -532,7 +532,7 @@ const DefaultTestSchema = z providers: EvalTargetsSchema.optional(), prompts: PromptsSchema.optional(), provider_output: ExpectedOutputSchema.optional(), - expected_output: ExpectedOutputSchema.optional(), + expected_output: z.never().optional(), assert: z.array(AssertionItemSchema).optional(), assert_scoring_function: z.union([z.string().min(1), JsonObjectSchema]).optional(), options: JsonObjectSchema.optional(), @@ -582,7 +582,7 @@ const EvalTestSchema = z.object({ provider_output: ExpectedOutputSchema.optional(), input: z.never().optional(), input_files: z.array(z.string()).optional(), - expected_output: ExpectedOutputSchema.optional(), + expected_output: z.never().optional(), assert: z.array(AssertionItemSchema).optional(), assert_scoring_function: z.union([z.string().min(1), JsonObjectSchema]).optional(), options: JsonObjectSchema.optional(), @@ -738,6 +738,7 @@ export const EvalFileSchema: z.ZodType = z tags: TagsSchema.optional(), license: z.string().optional(), requires: z.object({ agentv: z.string().optional() }).optional(), + expected_output: z.never().optional(), // Suite-level input input: z.never().optional(), prompts: PromptsSchema.optional(), diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index c07056271..8a1708d95 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -142,6 +142,7 @@ const KNOWN_TOP_LEVEL_FIELDS = new Set([ 'tags', 'license', 'requires', + 'expected_output', 'input', 'input_files', 'prompts', @@ -185,7 +186,6 @@ const KNOWN_DEFAULT_TEST_FIELDS = new Set([ 'providers', 'prompts', 'provider_output', - 'expected_output', 'assert', 'assert_scoring_function', 'options', @@ -220,6 +220,10 @@ const KNOWN_TEST_EXECUTION_FIELDS = new Set([ /** Removed top-level fields with migration hints. */ const REMOVED_TOP_LEVEL_FIELDS = new Map([ + [ + 'expected_output', + "Top-level 'expected_output' has been removed from authored eval YAML. Put reference answers in default_test.vars.expected_output or tests[].vars.expected_output and consume them with an explicit assertion such as { type: 'llm-rubric', value: 'Matches the reference answer: {{ expected_output }}' }.", + ], [ 'input', "Top-level 'input' has been removed from authored eval YAML. Author prompt text or chat messages in top-level 'prompts' and put shared data in default_test.vars or per-row data in tests[].vars.", @@ -273,7 +277,6 @@ const KNOWN_TEST_FIELDS = new Set([ 'provider_output', 'input', 'input_files', - 'expected_output', 'assert', 'assert_scoring_function', 'options', @@ -639,36 +642,15 @@ export async function validateEvalFile(filePath: string): Promise 0 && - isObject(expectedOutputField[0]) && - 'role' in expectedOutputField[0] - ) { - validateMessages( - expectedOutputField, - `${location}.expected_output`, - absolutePath, - errors, - ); - } - // Otherwise it's treated as structured array content - valid - } else if (isObject(expectedOutputField)) { - // Object shorthand or single message - both are valid - } else { - errors.push({ - severity: 'error', - filePath: absolutePath, - location: `${location}.expected_output`, - message: "Invalid 'expected_output' field (must be a string, object, or array)", - }); - } + errors.push({ + severity: 'error', + filePath: absolutePath, + location: `${location}.expected_output`, + message: + "tests[].expected_output has been removed from authored eval YAML. Put the reference answer in tests[].vars.expected_output and consume it with an explicit assertion, for example { type: 'llm-rubric', value: 'Matches the reference answer: {{ expected_output }}' }.", + }); } validateAssertArray( @@ -1337,11 +1319,21 @@ function validateDefaultTest( filePath, location: `default_test.${key}`, message: - 'Invalid default_test field. Supported fields: vars, provider, providers, prompts, provider_output, expected_output, assert, assert_scoring_function, options, threshold, metadata.', + 'Invalid default_test field. Supported fields: vars, provider, providers, prompts, provider_output, assert, assert_scoring_function, options, threshold, metadata.', }); } } + if (defaultTest.expected_output !== undefined) { + errors.push({ + severity: 'error', + filePath, + location: 'default_test.expected_output', + message: + "default_test.expected_output has been removed from authored eval YAML. Put shared reference answers in default_test.vars.expected_output and consume them with an explicit assertion, for example { type: 'llm-rubric', value: 'Matches the reference answer: {{ expected_output }}' }.", + }); + } + validateAssertArray( defaultTest.assert, 'default_test.assert', diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 1a04143e3..ee16157d3 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -127,6 +127,8 @@ type LoadOptions = { readonly category?: string; /** Internal DFS stack for detecting circular `type: suite` imports. */ readonly suiteImportStack?: readonly SuiteImportStackEntry[]; + /** Internal TS SDK bridge compatibility; authored YAML files must keep the default strict mode. */ + readonly allowInternalExpectedOutput?: boolean; }; type SuiteImportStackEntry = { @@ -1154,7 +1156,7 @@ export async function loadTestSuite( return loadTsEvalSuite(evalFilePath, resolveToAbsolutePath(repoRoot), options); } const { tests, parsed } = await loadTestsFromYaml(evalFilePath, repoRoot, options); - return buildEvalSuiteResult(parsed, tests); + return buildEvalSuiteResult(parsed, tests, options); } /** @deprecated Use `loadTestSuite` instead */ @@ -1173,7 +1175,7 @@ export async function loadTestSuiteFromYamlObject( options, ); - return buildEvalSuiteResult(parsed, tests); + return buildEvalSuiteResult(parsed, tests, options); } export async function loadTests( @@ -1246,6 +1248,9 @@ async function loadTestsFromParsedYamlValue( } rejectAuthoredWorkers(interpolated); rejectAuthoredDirectInput(interpolated); + if (options?.allowInternalExpectedOutput !== true) { + rejectAuthoredExpectedOutput(interpolated); + } const rawSuite = rawParsed as RawTestSuite; const resolvedDefaultTest = await resolveDefaultTestValue( @@ -1687,8 +1692,15 @@ async function loadTestsFromParsedYamlValue( }; } -function buildEvalSuiteResult(parsed: JsonObject, tests: readonly EvalTest[]): EvalSuiteResult { +function buildEvalSuiteResult( + parsed: JsonObject, + tests: readonly EvalTest[], + options?: LoadOptions, +): EvalSuiteResult { rejectAuthoredWorkers(parsed); + if (options?.allowInternalExpectedOutput !== true) { + rejectAuthoredExpectedOutput(parsed); + } const metadata = parseMetadata(parsed); const failOnError = extractFailOnError(parsed); const threshold = extractThreshold(parsed); @@ -1799,6 +1811,34 @@ function rejectAuthoredDirectInput(parsed: JsonObject): void { } } +function rejectAuthoredExpectedOutput(parsed: JsonObject): void { + if (parsed.expected_output !== undefined) { + throw new Error( + "Top-level 'expected_output' has been removed from authored eval YAML. Put reference answers in default_test.vars.expected_output or tests[].vars.expected_output and consume them with an explicit assertion such as { type: 'llm-rubric', value: 'Matches the reference answer: {{ expected_output }}' }.", + ); + } + + if (isJsonObject(parsed.default_test) && parsed.default_test.expected_output !== undefined) { + throw new Error( + "default_test.expected_output has been removed from authored eval YAML. Put shared reference answers in default_test.vars.expected_output and consume them with an explicit assertion such as { type: 'llm-rubric', value: 'Matches the reference answer: {{ expected_output }}' }.", + ); + } + + if (!Array.isArray(parsed.tests)) { + return; + } + + for (let index = 0; index < parsed.tests.length; index++) { + const entry = parsed.tests[index]; + if (!isJsonObject(entry) || entry.expected_output === undefined) { + continue; + } + throw new Error( + `tests[${index}].expected_output has been removed from authored eval YAML. Put the reference answer in tests[].vars.expected_output and consume it with an explicit assertion such as { type: 'llm-rubric', value: 'Matches the reference answer: {{ expected_output }}' }.`, + ); + } +} + function collectWorkersLocations(raw: unknown, location: string, locations: string[]): void { if (!isJsonObject(raw)) { return; diff --git a/packages/core/test/evaluation/conversation-mode.test.ts b/packages/core/test/evaluation/conversation-mode.test.ts index 8e46de93f..6cd477995 100644 --- a/packages/core/test/evaluation/conversation-mode.test.ts +++ b/packages/core/test/evaluation/conversation-mode.test.ts @@ -806,9 +806,9 @@ tests: - id: t1 criteria: Goal mode: conversation + expected_output: some output turns: - input: Turn 1 - expected_output: some output vars: input: hello `, @@ -816,9 +816,7 @@ tests: const result = await validateEvalFile(filePath); expect(result.valid).toBe(false); expect( - result.errors.some((e) => - e.message.includes("'expected_output' is not allowed with mode: conversation"), - ), + result.errors.some((e) => e.message.includes('tests[].expected_output has been removed')), ).toBe(true); }); diff --git a/packages/core/test/evaluation/criteria-optional.test.ts b/packages/core/test/evaluation/criteria-optional.test.ts index 217a5c320..14de76c55 100644 --- a/packages/core/test/evaluation/criteria-optional.test.ts +++ b/packages/core/test/evaluation/criteria-optional.test.ts @@ -5,7 +5,7 @@ import path from 'node:path'; import { loadTests } from '../../src/evaluation/yaml-parser.js'; -describe('criteria is optional when expected_output or assertions is present', () => { +describe('criteria is optional when assertions are present', () => { let tempDir: string; beforeAll(async () => { @@ -17,19 +17,19 @@ describe('criteria is optional when expected_output or assertions is present', ( await rm(tempDir, { recursive: true, force: true }); }); - it('accepts test with expected_output and no criteria', async () => { + it('accepts test with vars.expected_output when an assertion consumes it', async () => { await writeFile( path.join(tempDir, 'expected-output.eval.yaml'), `prompts: - "{{ input }}" tests: - id: test-01 - expected_output: sample expected output assert: - type: contains - value: sample + value: "{{ expected_output }}" vars: input: sample prompt + expected_output: sample expected output `, ); @@ -37,6 +37,10 @@ tests: expect(tests).toHaveLength(1); expect(tests[0].id).toBe('test-01'); expect(tests[0].criteria).toBe(''); + expect(tests[0].assertions?.[0]).toMatchObject({ + type: 'contains', + value: 'sample expected output', + }); }); it('accepts test with assertions only and no criteria', async () => { @@ -76,7 +80,7 @@ tests: expect(tests).toHaveLength(0); }); - it('skips test with no criteria, no expected_output, and no assertions', async () => { + it('skips test with no criteria and no assertions', async () => { await writeFile( path.join(tempDir, 'no-eval-spec.eval.yaml'), `prompts: @@ -92,6 +96,23 @@ tests: expect(tests).toHaveLength(0); }); + it('does not treat vars.expected_output as an evaluation spec by itself', async () => { + await writeFile( + path.join(tempDir, 'vars-reference-only.eval.yaml'), + `prompts: + - "{{ input }}" +tests: + - id: test-06 + vars: + input: sample prompt + expected_output: sample expected output +`, + ); + + const tests = await loadTests(path.join(tempDir, 'vars-reference-only.eval.yaml'), tempDir); + expect(tests).toHaveLength(0); + }); + it('accepts test with criteria (original behavior)', async () => { await writeFile( path.join(tempDir, 'with-criteria.eval.yaml'), diff --git a/packages/core/test/evaluation/eval-inline-experiment.test.ts b/packages/core/test/evaluation/eval-inline-experiment.test.ts index 8937f7daf..109aa9c15 100644 --- a/packages/core/test/evaluation/eval-inline-experiment.test.ts +++ b/packages/core/test/evaluation/eval-inline-experiment.test.ts @@ -257,7 +257,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ' - id: docs', ' vars:', ' topic: release notes', - ' expected_output: concise release-note summary', + ' criteria: Writes a concise release-note summary', '', ].join('\n'), ); @@ -305,12 +305,12 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ' - id: inherited-defaults', ' vars:', ' topic: release notes', - ' expected_output: concise release-note summary', + ' criteria: Writes a concise release-note summary', ' - id: overrides-default', ' vars:', ' audience: executives', ' topic: migration plan', - ' expected_output: executive migration summary', + ' criteria: Writes an executive migration summary', '', ].join('\n'), ); @@ -347,13 +347,13 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ' vars:', ' question: How do I reset my password?', ' input: "Answer in a {{ tone }} {{ category }} style: {{ question }}"', - ' expected_output: password reset guidance', + ' criteria: Gives password reset guidance', ' - id: direct-override', ' vars:', ' category: onboarding', ' question: Where is the getting started guide?', ' input: "Answer in a {{ tone }} {{ category }} style: {{ question }}"', - ' expected_output: getting started guidance', + ' criteria: Gives getting started guidance', ].join('\n'), ); @@ -384,7 +384,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ' - id: docs', ' vars:', ' topic: release notes', - ' expected_output: concrete release-note explanation', + ' criteria: Gives a concrete release-note explanation', '', ].join('\n'), ); diff --git a/packages/core/test/evaluation/interpolation-integration.test.ts b/packages/core/test/evaluation/interpolation-integration.test.ts index b77efe264..a5f191bdb 100644 --- a/packages/core/test/evaluation/interpolation-integration.test.ts +++ b/packages/core/test/evaluation/interpolation-integration.test.ts @@ -144,11 +144,9 @@ describe('env interpolation in YAML loading', () => { it('resolves missing variables to empty string', async () => { const evalFile = path.join(testDir, 'interp-missing.eval.yaml'); - // Include expected_output so the test is not skipped for missing criteria - // (empty criteria alone causes the test loader to skip it as incomplete) await writeFile( evalFile, - 'prompts:\n - "{{ input }}"\ntests:\n - id: test-1\n criteria: "prefix {{ env.AGENTV_NONEXISTENT_VAR }} suffix"\n expected_output: "some output"\n vars:\n input: "hello"\n', + 'prompts:\n - "{{ input }}"\ntests:\n - id: test-1\n criteria: "prefix {{ env.AGENTV_NONEXISTENT_VAR }} suffix"\n vars:\n input: "hello"\n', ); const cases = await loadTests(evalFile, testDir); expect(cases[0].criteria).toBe('prefix suffix'); diff --git a/packages/core/test/evaluation/loaders/jsonl-parser.test.ts b/packages/core/test/evaluation/loaders/jsonl-parser.test.ts index 9a8b89ca0..4717066c8 100644 --- a/packages/core/test/evaluation/loaders/jsonl-parser.test.ts +++ b/packages/core/test/evaluation/loaders/jsonl-parser.test.ts @@ -434,7 +434,7 @@ tests: expect(cases[0].assertions?.[0]?.rubrics?.[0]?.outcome).toBe('Goal'); }); - it('keeps expected_output-only YAML cases passive without implicit assertions', async () => { + it('keeps vars.expected_output-only YAML cases passive without implicit assertions', async () => { const yamlPath = path.join(tempDir, 'expected-output-only.yaml'); await writeFile( yamlPath, @@ -442,18 +442,56 @@ tests: - "{{ input }}" tests: - id: expected-only - expected_output: Reference answer vars: input: Query + expected_output: Reference answer `, ); const cases = await loadTests(yamlPath, tempDir); - expect(cases).toHaveLength(1); - expect(cases[0].criteria).toBe(''); - expect(cases[0].expected_output[0].content).toBe('Reference answer'); - expect(cases[0].assertions).toBeUndefined(); + expect(cases).toHaveLength(0); + }); + + it('rejects top-level authored YAML expected_output', async () => { + const yamlPath = path.join(tempDir, 'top-level-expected-output.yaml'); + await writeFile( + yamlPath, + `expected_output: Shared reference +prompts: + - "{{ input }}" +tests: + - id: expected-only + criteria: Goal + vars: + input: Query +`, + ); + + await expect(loadTests(yamlPath, tempDir)).rejects.toThrow( + "Top-level 'expected_output' has been removed", + ); + }); + + it('rejects default_test authored YAML expected_output', async () => { + const yamlPath = path.join(tempDir, 'default-test-expected-output.yaml'); + await writeFile( + yamlPath, + `default_test: + expected_output: Shared reference +prompts: + - "{{ input }}" +tests: + - id: expected-only + criteria: Goal + vars: + input: Query +`, + ); + + await expect(loadTests(yamlPath, tempDir)).rejects.toThrow( + 'default_test.expected_output has been removed', + ); }); it('accepts direct input shorthand without deprecation warnings', async () => { @@ -726,7 +764,7 @@ tests: expect(cases[0].input[1].role).toBe('user'); }); - it('supports expected_output string shorthand', async () => { + it('rejects authored YAML expected_output string shorthand', async () => { const yamlPath = path.join(tempDir, 'expected-string.yaml'); await writeFile( yamlPath, @@ -741,15 +779,12 @@ tests: `, ); - const cases = await loadTests(yamlPath, tempDir); - - expect(cases).toHaveLength(1); - expect(cases[0].expected_output).toHaveLength(1); - expect(cases[0].expected_output[0].role).toBe('assistant'); - expect(cases[0].expected_output[0].content).toBe('The answer is 4'); + await expect(loadTests(yamlPath, tempDir)).rejects.toThrow( + 'tests[0].expected_output has been removed', + ); }); - it('supports expected_output object shorthand', async () => { + it('rejects authored YAML expected_output object shorthand', async () => { const yamlPath = path.join(tempDir, 'expected-object.yaml'); await writeFile( yamlPath, @@ -766,13 +801,9 @@ tests: `, ); - const cases = await loadTests(yamlPath, tempDir); - - expect(cases).toHaveLength(1); - expect(cases[0].expected_output).toHaveLength(1); - expect(cases[0].expected_output[0].role).toBe('assistant'); - const content = cases[0].expected_output[0].content as { riskLevel: string }; - expect(content.riskLevel).toBe('High'); + await expect(loadTests(yamlPath, tempDir)).rejects.toThrow( + 'tests[0].expected_output has been removed', + ); }); it('resolves input message array from YAML', async () => { @@ -799,7 +830,7 @@ tests: }); describe('Mixed canonical and alias usage', () => { - it('allows mixing canonical and alias in same file', async () => { + it('rejects mixing authored YAML expected_output forms', async () => { const yamlPath = path.join(tempDir, 'mixed.yaml'); await writeFile( yamlPath, @@ -823,17 +854,12 @@ tests: `, ); - const cases = await loadTests(yamlPath, tempDir); - - expect(cases).toHaveLength(2); - expect(cases[0].id).toBe('test-canonical'); - expect(cases[0].input[0].content).toBe('Using canonical'); - expect(cases[1].id).toBe('test-alias'); - expect(cases[1].input[0].content).toBe('Using alias shorthand'); - expect(cases[1].expected_output[0].content).toBe('Alias response'); + await expect(loadTests(yamlPath, tempDir)).rejects.toThrow( + 'tests[0].expected_output has been removed', + ); }); - it('YAML and JSONL aliases produce equivalent results', async () => { + it('supports expected_output in JSONL raw cases but rejects authored YAML', async () => { const yamlPath = path.join(tempDir, 'equiv-alias.yaml'); const jsonlPath = path.join(tempDir, 'equiv-alias.jsonl'); @@ -856,21 +882,14 @@ tests: '{"id": "test-1", "criteria": "Goal", "input": "What is 2+2?", "expected_output": {"answer": 4}}\n', ); - const yamlCases = await loadTests(yamlPath, tempDir); + await expect(loadTests(yamlPath, tempDir)).rejects.toThrow( + 'tests[0].expected_output has been removed', + ); const jsonlCases = await loadTests(jsonlPath, tempDir); - expect(yamlCases).toHaveLength(1); expect(jsonlCases).toHaveLength(1); - - // Input should match - expect(jsonlCases[0].input[0].role).toBe(yamlCases[0].input[0].role); - expect(jsonlCases[0].input[0].content).toBe(yamlCases[0].input[0].content); - - // Expected output should match - expect(jsonlCases[0].expected_output[0].role).toBe(yamlCases[0].expected_output[0].role); - const yamlContent = yamlCases[0].expected_output[0].content as { answer: number }; const jsonlContent = jsonlCases[0].expected_output[0].content as { answer: number }; - expect(jsonlContent.answer).toBe(yamlContent.answer); + expect(jsonlContent.answer).toBe(4); }); }); }); diff --git a/packages/core/test/evaluation/suite-level-input.test.ts b/packages/core/test/evaluation/suite-level-input.test.ts index e1dee66c2..34fc2dbc8 100644 --- a/packages/core/test/evaluation/suite-level-input.test.ts +++ b/packages/core/test/evaluation/suite-level-input.test.ts @@ -309,7 +309,6 @@ tests: - id: templated vars: question: What is the capital of France? - expected_answer: Paris input: - role: user content: "Answer clearly: {{question}}" @@ -319,8 +318,11 @@ tests: content: Thinking about {{question}} - role: user content: Final answer only. + expected_output: Paris criteria: Answers {{question}} correctly - expected_output: "{{expected_answer}}" + assert: + - type: equals + value: "{{ expected_output }}" metadata: untouched: "{{question}}" `, @@ -343,7 +345,8 @@ tests: role: 'assistant', content: 'Thinking about What is the capital of France?', }); - expect(tests[0].expected_output).toEqual([{ role: 'assistant', content: 'Paris' }]); + expect(tests[0].expected_output).toEqual([]); + expect(tests[0].assertions?.[0]).toMatchObject({ type: 'equals', value: 'Paris' }); expect(tests[0].metadata).toEqual({ untouched: '{{question}}' }); }); diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index db4ea01bb..a6cbd6f09 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -1308,7 +1308,7 @@ tests: ); }); - it('validates eval file with expected_output alias string shorthand', async () => { + it('rejects eval file with test-level expected_output', async () => { const filePath = path.join(tempDir, 'output-string.yaml'); await writeFile( filePath, @@ -1325,24 +1325,63 @@ tests: const result = await validateEvalFile(filePath); - expect(result.valid).toBe(true); - expect(result.errors).toHaveLength(0); + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'tests[0].expected_output', + message: expect.stringContaining('tests[].expected_output has been removed'), + }), + ); }); - it('validates eval file with expected_output alias object shorthand', async () => { + it('rejects eval file with default_test expected_output', async () => { const filePath = path.join(tempDir, 'output-object.yaml'); await writeFile( filePath, - `prompts: + `default_test: + expected_output: Shared reference +prompts: + - "{{ prompt }}" +tests: + - id: test-1 + vars: + prompt: Query + assert: + - type: llm-rubric + value: "Matches the reference answer: {{ expected_output }}" +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'default_test.expected_output', + message: expect.stringContaining('default_test.expected_output has been removed'), + }), + ); + }); + + it('validates vars.expected_output when an explicit assertion consumes it', async () => { + const filePath = path.join(tempDir, 'vars-expected-output.yaml'); + await writeFile( + filePath, + `default_test: + assert: + - type: llm-rubric + value: "Matches the reference answer: {{ expected_output }}" +prompts: - "{{ prompt }}" tests: - id: test-1 - criteria: Goal vars: prompt: Query - expected_output: - riskLevel: High - confidence: 0.95 + expected_output: + riskLevel: High + confidence: 0.95 `, ); @@ -1451,7 +1490,6 @@ tests: expected: answer: "4" criteria: "Answers {{question}} correctly" - expected_output: "{{expected.answer}}" `, ); @@ -2453,7 +2491,8 @@ tests: - id: test-1 input: "Hello" criteria: Some criteria - expected_output: "World" + vars: + expected_output: "World" assert: - type: contains value: "world" diff --git a/scripts/migrate-hard-deprecations.test.ts b/scripts/migrate-hard-deprecations.test.ts index 8ff749a97..e2af233dc 100644 --- a/scripts/migrate-hard-deprecations.test.ts +++ b/scripts/migrate-hard-deprecations.test.ts @@ -129,4 +129,76 @@ tests: expect(secondAssertion.transform).toContain('return (() =>'); expect(secondAssertion.transform).toContain('Bun.spawnSync(["node","xlsx.js"]'); }); + + it('migrates authored expected_output to vars.expected_output with explicit rubric assertion', () => { + const migrated = migrateSnippet(`description: legacy expected output +prompts: + - "{{ input }}" +tests: + - id: one + vars: + input: What is 2+2? + expected_output: "4" +`); + const parsed = asRecord(parse(migrated)); + const tests = parsed.tests as Array>; + const firstTest = asRecord(tests[0]); + const vars = asRecord(firstTest.vars); + const assertions = firstTest.assert as Array>; + + expect(firstTest.expected_output).toBeUndefined(); + expect(vars.expected_output).toBe('4'); + expect(assertions).toEqual([ + { + type: 'llm-rubric', + value: 'Matches the reference answer: {{ expected_output }}', + }, + ]); + }); + + it('preserves explicit assert strategy when migrating expected_output', () => { + const migrated = migrateSnippet(`prompts: + - "{{ input }}" +tests: + - id: one + vars: + input: What is 2+2? + expected_output: "4" + assert: + - type: equals + value: "{{ expected_output }}" +`); + const parsed = asRecord(parse(migrated)); + const tests = parsed.tests as Array>; + const firstTest = asRecord(tests[0]); + const vars = asRecord(firstTest.vars); + const assertions = firstTest.assert as Array>; + + expect(firstTest.expected_output).toBeUndefined(); + expect(vars.expected_output).toBe('4'); + expect(assertions).toEqual([{ type: 'equals', value: '{{ expected_output }}' }]); + }); + + it('migrates default_test expected_output to default_test.vars.expected_output', () => { + const migrated = migrateSnippet(`default_test: + expected_output: Paris +prompts: + - "{{ input }}" +tests: + - id: one + vars: + input: What is the capital of France? +`); + const parsed = asRecord(parse(migrated)); + const defaultTest = asRecord(parsed.default_test); + const vars = asRecord(defaultTest.vars); + const assertions = defaultTest.assert as Array>; + + expect(defaultTest.expected_output).toBeUndefined(); + expect(vars.expected_output).toBe('Paris'); + expect(assertions[0]).toEqual({ + type: 'llm-rubric', + value: 'Matches the reference answer: {{ expected_output }}', + }); + }); }); diff --git a/scripts/migrate-hard-deprecations.ts b/scripts/migrate-hard-deprecations.ts index 6f4d8bdb0..a99e58d21 100644 --- a/scripts/migrate-hard-deprecations.ts +++ b/scripts/migrate-hard-deprecations.ts @@ -11,6 +11,11 @@ const ROOTS = [ 'apps/cli/test/commands/eval/pipeline/fixtures', ]; const INPUT_PROMPT = '{{ input }}'; +const EXPECTED_OUTPUT_VAR = 'expected_output'; +const REFERENCE_MATCH_ASSERTION = { + type: 'llm-rubric', + value: 'Matches the reference answer: {{ expected_output }}', +}; const LEGACY_ENV_PATTERN = /\$\{\{\s*([A-Z_][A-Z0-9_]*)\s*\}\}/g; const PREPROCESSOR_MEDIA_TYPES: Readonly> = { xlsx: ['xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', '.xlsx'], @@ -352,12 +357,47 @@ function setVarsInput(testCase: JsonObject, input: unknown): void { testCase.vars = vars; } -function migrateCase(testCase: JsonObject, fileDir: string, suiteInput?: unknown): boolean { +function setVarsExpectedOutput(testCase: JsonObject, expectedOutput: unknown): void { + const vars = isObject(testCase.vars) ? testCase.vars : {}; + vars[EXPECTED_OUTPUT_VAR] = expectedOutput; + testCase.vars = vars; +} + +function hasExplicitAssertStrategy(value: JsonObject): boolean { + return ( + Object.hasOwn(value, 'assert') || + (typeof value.criteria === 'string' && value.criteria.trim().length > 0) || + (typeof value.expected_outcome === 'string' && value.expected_outcome.trim().length > 0) + ); +} + +function addReferenceAssertion(value: JsonObject): void { + value.assert = [clone(REFERENCE_MATCH_ASSERTION)]; +} + +function migrateExpectedOutput(value: JsonObject, hasInheritedAssertStrategy: boolean): boolean { + if (!Object.hasOwn(value, 'expected_output')) return false; + const hadExplicitAssertStrategy = hasExplicitAssertStrategy(value) || hasInheritedAssertStrategy; + setVarsExpectedOutput(value, value.expected_output); + Reflect.deleteProperty(value, 'expected_output'); + if (!hadExplicitAssertStrategy) { + addReferenceAssertion(value); + } + return true; +} + +function migrateCase( + testCase: JsonObject, + fileDir: string, + suiteInput?: unknown, + hasInheritedAssertStrategy = false, +): boolean { + let changed = migrateExpectedOutput(testCase, hasInheritedAssertStrategy); const hasCaseInput = Object.hasOwn(testCase, 'input'); const hasCaseInputFiles = Object.hasOwn(testCase, 'input_files'); const effectiveSuiteInput = hasSkipDefaults(testCase) ? undefined : suiteInput; if (!hasCaseInput && !hasCaseInputFiles && effectiveSuiteInput === undefined) { - return false; + return changed; } const caseInput = hasCaseInputFiles @@ -369,6 +409,47 @@ function migrateCase(testCase: JsonObject, fileDir: string, suiteInput?: unknown } Reflect.deleteProperty(testCase, 'input'); Reflect.deleteProperty(testCase, 'input_files'); + changed = true; + return changed; +} + +function ensureDefaultTest(suite: JsonObject): JsonObject { + if (isObject(suite.default_test)) return suite.default_test; + const defaultTest: JsonObject = {}; + suite.default_test = defaultTest; + return defaultTest; +} + +function migrateSuiteExpectedOutput( + suite: JsonObject, + hasInheritedAssertStrategy: boolean, +): boolean { + if (!Object.hasOwn(suite, 'expected_output')) return false; + const defaultTest = ensureDefaultTest(suite); + const hadExplicitAssertStrategy = + hasExplicitAssertStrategy(defaultTest) || + hasExplicitAssertStrategy(suite) || + hasInheritedAssertStrategy; + setVarsExpectedOutput(defaultTest, suite.expected_output); + Reflect.deleteProperty(suite, 'expected_output'); + if (!hadExplicitAssertStrategy) { + addReferenceAssertion(defaultTest); + } + return true; +} + +function migrateDefaultTestExpectedOutput(suite: JsonObject): boolean { + if (!isObject(suite.default_test) || !Object.hasOwn(suite.default_test, 'expected_output')) { + return false; + } + const defaultTest = suite.default_test; + const hadExplicitAssertStrategy = + hasExplicitAssertStrategy(defaultTest) || hasExplicitAssertStrategy(suite); + setVarsExpectedOutput(defaultTest, defaultTest.expected_output); + Reflect.deleteProperty(defaultTest, 'expected_output'); + if (!hadExplicitAssertStrategy) { + addReferenceAssertion(defaultTest); + } return true; } @@ -482,6 +563,8 @@ function migrateYamlValue( const isSuiteLike = value.tests !== undefined || value.eval_cases !== undefined || value.imports !== undefined; + changed = migrateSuiteExpectedOutput(value, false) || changed; + changed = migrateDefaultTestExpectedOutput(value) || changed; if (!isSuiteLike && inheritedSuiteInput === undefined) { return changed; } @@ -492,13 +575,16 @@ function migrateYamlValue( : inheritedSuiteInput; const rawTests = value.tests ?? value.eval_cases; + const hasSuiteAssertStrategy = + hasExplicitAssertStrategy(value) || + (isObject(value.default_test) && hasExplicitAssertStrategy(value.default_test)); if (Array.isArray(rawTests)) { for (const entry of rawTests) { if (isObject(entry) && !Object.hasOwn(entry, 'include')) { changed = migrateWorkspace(entry.workspace) || changed; changed = migrateOptionsPostprocess(entry.options) || changed; changed = migrateAssertions(entry.assert) || changed; - changed = migrateCase(entry, fileDir, suiteInput) || changed; + changed = migrateCase(entry, fileDir, suiteInput, hasSuiteAssertStrategy) || changed; } } } else if (typeof rawTests === 'string') { @@ -535,6 +621,7 @@ function migrateYamlSnippet(source: string, filePath: string): string | undefine if ( !source.includes('input:') && !source.includes('input_files:') && + !source.includes('expected_output:') && !source.includes('${{') && !source.includes('execution:') && !source.includes('preprocessors:') && diff --git a/skills-data/agentv-eval-migrations/references/breaking-changes.md b/skills-data/agentv-eval-migrations/references/breaking-changes.md index 9c69e9404..cb768caff 100644 --- a/skills-data/agentv-eval-migrations/references/breaking-changes.md +++ b/skills-data/agentv-eval-migrations/references/breaking-changes.md @@ -142,34 +142,40 @@ for authored eval YAML and assertion template YAML. preferred way to express the whole semantic contract. Put actual grading checks in `assert`, usually as plain strings. -## `criteria` Is Optional, `expected_output` Is Passive +## `criteria` Is Optional, Reference Answers Live In Vars ### v4.42.4 Shape -v4.42.4 docs treated `criteria` as required, and when no `assertions` were -present a default `llm-grader` evaluated the case against `criteria`: +v4.42.4 docs treated `criteria` as required. Older AgentV YAML also allowed +`expected_output` as a sibling field on test cases: ```yaml tests: - id: simple-eval criteria: Assistant correctly explains the bug and proposes a fix input: "Debug this function..." + expected_output: The answer explains the root cause and fix. ``` ### Current Shape Current docs and schema make `criteria` optional. Authored graders live under -`assert`. Plain strings in `assert` become an `llm-rubric` check. `expected_output` -is reference data available to graders; by itself it does not choose a grader. +`assert`. Plain strings in `assert` become an `llm-rubric` check. +Promptfoo-aligned reference answers live in `vars.expected_output`, and only +affect grading when an explicit assertion consumes them. ```yaml +default_test: + assert: + - type: llm-rubric + value: "Matches the reference answer: {{ expected_output }}" +prompts: + - "{{ input }}" tests: - id: simple-eval - input: "Debug this function..." - expected_output: The answer explains the root cause and fix. - assert: - - Assistant correctly explains the bug - - Assistant proposes a concrete fix + vars: + input: "Debug this function..." + expected_output: The answer explains the root cause and fix. ``` ### Migration Steps @@ -178,14 +184,18 @@ tests: `assert` as one or more plain strings. - Keep `criteria` only when multiple graders need shared context that is not itself the asserted checklist. -- If `expected_output` was being used as "the rubric", add explicit `assert` - entries that state how the reference should be used. +- Move `tests[].expected_output` or `default_test.expected_output` to + `vars.expected_output`. +- Add or keep an explicit assertion strategy. Use `llm-rubric` with + `{{ expected_output }}` for semantic reference-answer checks, or deterministic + assertion `value: "{{ expected_output }}"` when the grader type compares a + concrete value. ### Verification ```bash bun apps/cli/src/cli.ts validate path/to/eval.eval.yaml -rg -n "criteria:" path/to/evals +rg -n "expected_output:" path/to/evals ``` For any remaining `criteria`, confirm it is shared grader context and not a diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index ecc32f844..6b15cfad3 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -54,6 +54,9 @@ }, "additionalProperties": false }, + "expected_output": { + "not": {} + }, "input": { "not": {} }, @@ -2013,59 +2016,7 @@ } }, "expected_output": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "role": { - "type": "string", - "enum": ["system", "user", "assistant", "tool"] - }, - "content": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["text", "file", "image"] - }, - "value": { - "type": "string" - } - }, - "required": ["type", "value"], - "additionalProperties": false - } - } - ] - } - }, - "required": ["role", "content"], - "additionalProperties": false - } - } - ] + "not": {} }, "assert": { "type": "array", @@ -3993,59 +3944,7 @@ } }, "expected_output": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "role": { - "type": "string", - "enum": ["system", "user", "assistant", "tool"] - }, - "content": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["text", "file", "image"] - }, - "value": { - "type": "string" - } - }, - "required": ["type", "value"], - "additionalProperties": false - } - } - ] - } - }, - "required": ["role", "content"], - "additionalProperties": false - } - } - ] + "not": {} }, "assert": { "type": "array", @@ -7036,59 +6935,7 @@ ] }, "expected_output": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "role": { - "type": "string", - "enum": ["system", "user", "assistant", "tool"] - }, - "content": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["text", "file", "image"] - }, - "value": { - "type": "string" - } - }, - "required": ["type", "value"], - "additionalProperties": false - } - } - ] - } - }, - "required": ["role", "content"], - "additionalProperties": false - } - } - ] + "not": {} }, "assert": { "type": "array", @@ -9609,59 +9456,7 @@ } }, "expected_output": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "role": { - "type": "string", - "enum": ["system", "user", "assistant", "tool"] - }, - "content": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["text", "file", "image"] - }, - "value": { - "type": "string" - } - }, - "required": ["type", "value"], - "additionalProperties": false - } - } - ] - } - }, - "required": ["role", "content"], - "additionalProperties": false - } - } - ] + "not": {} }, "assert": { "type": "array",