From 749e2eff397c10d8591af5f3cf8785ab0ce96e87 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 14 Apr 2026 07:40:39 +0000 Subject: [PATCH 1/4] fix: auto-weight grouped rubrics shorthand by criteria count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When string shorthand assertions are mixed with other explicit graders, the rubrics grader created from the strings now gets weight = number of criteria, making each user-visible assertion contribute equal weight to the overall score. Before: [contains, "A", "B", "C"] → contains(w=1) + rubrics(w=1) → 50/50 After: [contains, "A", "B", "C"] → contains(w=1) + rubrics(w=3) → 25/75 The shorthand abstraction is now transparent — users who write N string criteria alongside M explicit graders get equal weight per visible line, without needing to know about internal grader grouping. Closes #1098 Co-Authored-By: Claude Sonnet 4.6 --- .../evaluation/loaders/evaluator-parser.ts | 6 +- .../loaders/evaluator-parser.test.ts | 55 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/packages/core/src/evaluation/loaders/evaluator-parser.ts b/packages/core/src/evaluation/loaders/evaluator-parser.ts index 2e69b4639..6206c7027 100644 --- a/packages/core/src/evaluation/loaders/evaluator-parser.ts +++ b/packages/core/src/evaluation/loaders/evaluator-parser.ts @@ -288,7 +288,11 @@ async function parseEvaluatorList( } const placeholderIndex = result.indexOf(PLACEHOLDER); if (strings.length > 0 && placeholderIndex !== -1) { - result[placeholderIndex] = { type: 'rubrics', criteria: strings }; + // Set weight = number of criteria so each user-visible string assertion contributes + // equal weight to the overall score alongside other explicit graders. + // e.g. [contains, "crit1", "crit2", "crit3"] → contains(w=1) + rubrics(w=3) + // → each of the 4 visible assertions counts equally. + result[placeholderIndex] = { type: 'rubrics', criteria: strings, weight: strings.length }; } else if (placeholderIndex !== -1) { // All strings were empty — remove the placeholder result.splice(placeholderIndex, 1); diff --git a/packages/core/test/evaluation/loaders/evaluator-parser.test.ts b/packages/core/test/evaluation/loaders/evaluator-parser.test.ts index bf2998c58..b839dd2e7 100644 --- a/packages/core/test/evaluation/loaders/evaluator-parser.test.ts +++ b/packages/core/test/evaluation/loaders/evaluator-parser.test.ts @@ -1989,6 +1989,61 @@ describe('parseEvaluators - string shorthand in assertions', () => { expect(evaluators).toBeUndefined(); }); + + it('sets rubrics grader weight = criteria count when mixed with other graders', async () => { + // User sees 4 assertions; each should contribute equal weight. + // rubrics(w=3) + contains(w=1) → each visible assertion = 1/4. + const evaluators = await parseEvaluators( + { + assertions: [ + 'Identifies the undefined access', + 'Suggests a null-safe fix', + 'Explains why the original code is dangerous', + { type: 'contains', value: 'null' }, + ], + }, + undefined, + ['/tmp'], + 'test-id', + ); + + expect(evaluators).toHaveLength(2); + const rubrics = evaluators?.[0] as LlmGraderEvaluatorConfig; + expect(rubrics.type).toBe('llm-grader'); + expect(rubrics.rubrics).toHaveLength(3); + expect(rubrics.weight).toBe(3); + expect(evaluators?.[1].type).toBe('contains'); + expect(evaluators?.[1].weight).toBeUndefined(); // explicit graders keep their own weight + }); + + it('sets weight = 1 for a single string criterion mixed with another grader', async () => { + const evaluators = await parseEvaluators( + { + assertions: ['Response is polite', { type: 'contains', value: 'ok' }], + }, + undefined, + ['/tmp'], + 'test-id', + ); + + expect(evaluators).toHaveLength(2); + expect((evaluators?.[0] as LlmGraderEvaluatorConfig).weight).toBe(1); + }); + + it('sets weight = criteria count even when all assertions are strings', async () => { + // Weight on the sole grader has no effect on scoring but is set for consistency. + const evaluators = await parseEvaluators( + { + assertions: ['Criterion A', 'Criterion B', 'Criterion C'], + }, + undefined, + ['/tmp'], + 'test-id', + ); + + expect(evaluators).toHaveLength(1); + expect((evaluators?.[0] as LlmGraderEvaluatorConfig).weight).toBe(3); + }); }); describe('parseEvaluators - file:// prefix prompt resolution', () => { From 62fa5e15a9b41d8eb5eb11f73074df14abdd36c4 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 14 Apr 2026 07:42:15 +0000 Subject: [PATCH 2/4] style: fix biome formatting --- packages/core/src/evaluation/loaders/evaluator-parser.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/core/src/evaluation/loaders/evaluator-parser.ts b/packages/core/src/evaluation/loaders/evaluator-parser.ts index 6206c7027..8a878eee4 100644 --- a/packages/core/src/evaluation/loaders/evaluator-parser.ts +++ b/packages/core/src/evaluation/loaders/evaluator-parser.ts @@ -292,7 +292,11 @@ async function parseEvaluatorList( // equal weight to the overall score alongside other explicit graders. // e.g. [contains, "crit1", "crit2", "crit3"] → contains(w=1) + rubrics(w=3) // → each of the 4 visible assertions counts equally. - result[placeholderIndex] = { type: 'rubrics', criteria: strings, weight: strings.length }; + result[placeholderIndex] = { + type: 'rubrics', + criteria: strings, + weight: strings.length, + }; } else if (placeholderIndex !== -1) { // All strings were empty — remove the placeholder result.splice(placeholderIndex, 1); From e37fe40b8732a7db8702cf728c07301419b6deda Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 14 Apr 2026 12:34:20 +0000 Subject: [PATCH 3/4] test: remove redundant shorthand weight tests --- .../loaders/evaluator-parser.test.ts | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/packages/core/test/evaluation/loaders/evaluator-parser.test.ts b/packages/core/test/evaluation/loaders/evaluator-parser.test.ts index b839dd2e7..b0565649e 100644 --- a/packages/core/test/evaluation/loaders/evaluator-parser.test.ts +++ b/packages/core/test/evaluation/loaders/evaluator-parser.test.ts @@ -2016,34 +2016,6 @@ describe('parseEvaluators - string shorthand in assertions', () => { expect(evaluators?.[1].weight).toBeUndefined(); // explicit graders keep their own weight }); - it('sets weight = 1 for a single string criterion mixed with another grader', async () => { - const evaluators = await parseEvaluators( - { - assertions: ['Response is polite', { type: 'contains', value: 'ok' }], - }, - undefined, - ['/tmp'], - 'test-id', - ); - - expect(evaluators).toHaveLength(2); - expect((evaluators?.[0] as LlmGraderEvaluatorConfig).weight).toBe(1); - }); - - it('sets weight = criteria count even when all assertions are strings', async () => { - // Weight on the sole grader has no effect on scoring but is set for consistency. - const evaluators = await parseEvaluators( - { - assertions: ['Criterion A', 'Criterion B', 'Criterion C'], - }, - undefined, - ['/tmp'], - 'test-id', - ); - - expect(evaluators).toHaveLength(1); - expect((evaluators?.[0] as LlmGraderEvaluatorConfig).weight).toBe(3); - }); }); describe('parseEvaluators - file:// prefix prompt resolution', () => { From 20efd94d53d958fcbffe504529f40182e51fe53f Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 14 Apr 2026 12:35:45 +0000 Subject: [PATCH 4/4] style: fix trailing blank line --- packages/core/test/evaluation/loaders/evaluator-parser.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/core/test/evaluation/loaders/evaluator-parser.test.ts b/packages/core/test/evaluation/loaders/evaluator-parser.test.ts index b0565649e..e5cd4e571 100644 --- a/packages/core/test/evaluation/loaders/evaluator-parser.test.ts +++ b/packages/core/test/evaluation/loaders/evaluator-parser.test.ts @@ -2015,7 +2015,6 @@ describe('parseEvaluators - string shorthand in assertions', () => { expect(evaluators?.[1].type).toBe('contains'); expect(evaluators?.[1].weight).toBeUndefined(); // explicit graders keep their own weight }); - }); describe('parseEvaluators - file:// prefix prompt resolution', () => {