fix(core): align assert-set scoring semantics

apps/web/src/content/docs/docs/next/graders/assert-set.mdx

@@ -24,7 +24,9 @@ assert:
         weight: 0.6
 ```
 
-Child assertions run independently. The parent score is the weighted average of child scores. `threshold` defaults to `1`, so omit it when every child must pass.
+Child assertions run independently. The parent score is the weighted average of child scores.
+
+When `threshold` is omitted, the assert set passes only when every nonzero-weight child assertion passes. When `threshold` is present, the aggregate score determines the parent pass/fail verdict, so a partially failing set can pass if its weighted score meets the threshold.
 
 ## Patterns
 
@@ -56,6 +58,27 @@ assert:
         value: capital of france
 ```
 
+Share config across children:
+
+```yaml
+assert:
+  - metric: semantic_similarity
+    type: assert-set
+    config:
+      embedding_provider:
+        base_url: http://127.0.0.1:10531/v1
+        model: text-embedding-3-small
+    assert:
+      - metric: similarity
+        type: similar
+        value: Paris is the capital of France.
+      - metric: script_check
+        type: javascript
+        value: context.config.embedding_provider.model.length > 0
+```
+
+Parent `config` is inherited by child assertions. A child assertion can set its own `config` fields to override shared values.
+
 Nest `assert-set` only when the hierarchy helps review the result:
 
 ```yaml
@@ -99,4 +122,4 @@ An assert set returns nested child scores:
 
 ## Promptfoo Alignment
 
-AgentV uses Promptfoo's `type: assert-set` spelling for authored assertion groups. `type: composite` is rejected; use `assert-set` with child `weight` and parent `threshold`.
+AgentV uses Promptfoo's `type: assert-set` spelling for authored assertion groups. Public YAML uses nested `assert` entries, child `weight`, optional parent `threshold`, and optional parent `config`. `type: composite` is rejected; use `assert-set`.

apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx

@@ -40,7 +40,7 @@ implements equivalent semantics directly.
 | Target object identity | Provider options often use `id` for backend/provider spec and optional `label` for display or matching. | Target objects use stable `id` for target identity, `provider` for backend kind, optional `runtime`, and `config` for provider settings. | Keep AgentV divergence | AgentV does not copy Promptfoo's `label`/`id` baggage because `provider` already names the backend boundary. |
 | Direct input suites | Promptfoo prompt authoring normally goes through `prompts` plus vars. | `tests[].input`, top-level `input`, `tests[].input_files`, and top-level `input_files` are direct-input conveniences. | Keep AgentV extension | Do not mix these fields with top-level `prompts`. Use `tests[].vars` with `prompts`, or remove `prompts` for a direct-input AgentV suite. |
 | Suite assertions | `assert` entries can be strings or typed assertion objects. | `assert` entries can be strings, typed assertion objects, script graders, or AgentV extension graders. | Align with Promptfoo | Plain strings become semantic rubric checks. Use `assert`, not `assertions`, in current authored eval YAML. |
-| Assertion grouping | `type: assert-set` with child `assert` entries. | `type: assert-set` with child `assert`, weights, and parent threshold. | Align with Promptfoo | `type: composite` is rejected; use `assert-set`. |
+| Assertion grouping | `type: assert-set` with child `assert` entries, optional `config`, `metric`, `weight`, and `threshold`. | `type: assert-set` with child `assert`, optional `config`, metric names, weights, and parent threshold. | Align with Promptfoo | Parent `config` is inherited by child assertions; child `config` keys override shared parent keys. Without `threshold`, pass/fail follows nonzero-weight child assertions. With `threshold`, the weighted aggregate score determines pass/fail. `type: composite` is rejected; use `assert-set`. |
 | Deterministic assertion vocabulary | Common Promptfoo types include `contains`, `icontains`, `contains-any`, `contains-all`, `starts-with`, `regex`, `is-json`, `equals`, `latency`, `cost`, `javascript`, `python`, `webhook`, `similar`, and `llm-rubric`. | AgentV accepts the implemented overlap, including `contains`, `icontains`, `contains-any`, `contains-all`, `starts-with`, `regex`, `is-json`, `equals`, `latency`, `cost`, `javascript`, `python`, `webhook`, `similar`, and `llm-rubric`. | Align with Promptfoo | Unsupported Promptfoo assertion names error instead of silently becoming custom assertion names. |
 | Custom assertion terminology | Promptfoo calls normal eval custom logic assertions, with fixed code assertion types such as `javascript`, `python`, `ruby`, and `webhook`. | `defineAssertion()` files in `.agentv/assertions/` become reusable assertion type names. | Keep AgentV extension | AgentV keeps assertion terminology and extends discovery to arbitrary assertion type names such as `has-citation`. |
 | Script/custom grader terminology | Promptfoo custom code assertions are still assertion types. | `defineScriptGrader()` powers command-backed graders referenced with `type: script` and `command:`. | Keep AgentV divergence | Use script grader wording only for command-backed or LLM-backed scoring components that need explicit score and assertion-result control. |

packages/core/src/evaluation/graders/promptfoo-assertions.ts

@@ -22,7 +22,10 @@ type ScriptResult =
       readonly details?: JsonObject;
     };
 
-function buildAssertionContext(context: EvaluationContext): Record<string, unknown> {
+function buildAssertionContext(
+  context: EvaluationContext,
+  assertionConfig?: JsonObject,
+): Record<string, unknown> {
   return {
     criteria: context.evalCase.criteria,
     expectedOutput: context.evalCase.expected_output,
@@ -35,6 +38,7 @@ function buildAssertionContext(context: EvaluationContext): Record<string, unkno
     fileChanges: context.fileChanges ?? null,
     workspacePath: context.workspacePath ?? null,
     dependencyResults: context.dependencyResults ?? null,
+    ...(assertionConfig ? { config: assertionConfig } : {}),
   };
 }
 
@@ -111,7 +115,10 @@ export class JavascriptAssertionGrader implements Grader {
   async evaluate(context: EvaluationContext): Promise<EvaluationScore> {
     try {
       const fn = new Function('output', 'context', buildFunctionBody(this.config.value));
-      const result = (await fn(context.candidate, buildAssertionContext(context))) as ScriptResult;
+      const result = (await fn(
+        context.candidate,
+        buildAssertionContext(context, this.config.config),
+      )) as ScriptResult;
       return normalizeScriptResult(
         result,
         'Javascript assertion returned a failing result',
@@ -162,7 +169,9 @@ export class PythonAssertionGrader implements Grader {
   async evaluate(context: EvaluationContext): Promise<EvaluationScore> {
     const payload = JSON.stringify({
       output: context.candidate,
-      context: serializeSnakeCaseBoundaryPayload(buildAssertionContext(context)),
+      context: serializeSnakeCaseBoundaryPayload(
+        buildAssertionContext(context, this.config.config),
+      ),
     });
     try {
       const result = await execFileWithStdin(
@@ -205,7 +214,9 @@ export class WebhookAssertionGrader implements Grader {
         headers: { 'content-type': 'application/json' },
         body: JSON.stringify({
           output: context.candidate,
-          context: serializeSnakeCaseBoundaryPayload(buildAssertionContext(context)),
+          context: serializeSnakeCaseBoundaryPayload(
+            buildAssertionContext(context, this.config.config),
+          ),
         }),
       });
       if (!response.ok) {
@@ -242,13 +253,14 @@ export class AssertSetGrader implements Grader {
   async evaluate(context: EvaluationContext): Promise<EvaluationScore> {
     const scores = [];
     for (const childConfig of this.config.assertions) {
-      const child = await this.createChild(childConfig);
+      const resolvedChildConfig = withAssertSetConfig(childConfig, this.config.config);
+      const child = await this.createChild(resolvedChildConfig);
       const result = await child.evaluate(context);
       scores.push({
-        name: childConfig.name,
-        type: childConfig.type,
+        name: resolvedChildConfig.name,
+        type: resolvedChildConfig.type,
         score: result.score,
-        weight: childConfig.weight ?? 1,
+        weight: resolvedChildConfig.weight ?? 1,
         verdict: result.verdict,
         assertions: result.assertions,
         graderRawRequest: result.graderRawRequest,
@@ -261,19 +273,40 @@ export class AssertSetGrader implements Grader {
     const totalWeight = scores.reduce((sum, score) => sum + (score.weight ?? 1), 0) || 1;
     const score =
       scores.reduce((sum, item) => sum + item.score * (item.weight ?? 1), 0) / totalWeight;
-    const threshold = this.config.threshold ?? 1;
-    const passed = score >= threshold;
+    const threshold = this.config.threshold;
+    const passed =
+      threshold !== undefined
+        ? score >= threshold
+        : scores.every((item) => (item.weight ?? 1) === 0 || item.verdict === 'pass');
     return {
       score,
       verdict: passed ? 'pass' : 'fail',
       assertions: scores.flatMap((item) => item.assertions),
       expectedAspectCount: scores.reduce((sum, item) => sum + item.assertions.length, 0) || 1,
       scores,
-      details: { threshold },
+      ...(threshold !== undefined ? { details: { threshold } } : {}),
     };
   }
 }
 
+function withAssertSetConfig(
+  childConfig: AssertSetGraderConfig['assertions'][number],
+  parentConfig?: JsonObject,
+): AssertSetGraderConfig['assertions'][number] {
+  if (!parentConfig) {
+    return childConfig;
+  }
+
+  const existingConfig = (childConfig as { readonly config?: JsonObject }).config;
+  return {
+    ...childConfig,
+    config: {
+      ...parentConfig,
+      ...(existingConfig ?? {}),
+    },
+  } as AssertSetGraderConfig['assertions'][number];
+}
+
 function getEmbeddingConfig(config: SimilarGraderConfig): JsonObject | undefined {
   const provider = typeof config.provider === 'object' ? config.provider : undefined;
   const nested =

packages/core/src/evaluation/loaders/grader-parser.ts

@@ -455,6 +455,7 @@ async function parseGraderList(
   evalId: string,
   defaultPreprocessors?: readonly ContentPreprocessorConfig[],
   defaultRubricPrompt?: JsonValue,
+  inheritedAssertionConfig?: JsonObject,
 ): Promise<readonly GraderConfig[] | undefined> {
   const expandedEvaluators = await expandGraderEntries(candidateEvaluators, searchRoots, evalId);
   if (!expandedEvaluators) {
@@ -509,12 +510,13 @@ async function parseGraderList(
 
   const evaluators: GraderConfig[] = [];
 
-  for (const rawEvaluator of processedEvaluators) {
-    if (!isJsonObject(rawEvaluator)) {
+  for (const rawEvaluatorEntry of processedEvaluators) {
+    if (!isJsonObject(rawEvaluatorEntry)) {
       logWarning(`Skipping invalid evaluator entry for '${evalId}' (expected object)`);
       continue;
     }
 
+    const rawEvaluator = withInheritedAssertionConfig(rawEvaluatorEntry, inheritedAssertionConfig);
     const rawName = asString(rawEvaluator.metric);
     const rawType = rawEvaluator.type;
     const typeValue = typeof rawType === 'string' ? normalizeGraderType(rawType) : rawType;
@@ -592,12 +594,14 @@ async function parseGraderList(
         continue;
       }
 
+      const config = isJsonObject(rawEvaluator.config) ? rawEvaluator.config : undefined;
       const parsedMembers = await parseGraderList(
         rawMembers as JsonValue,
         searchRoots,
         `${evalId}:${name}`,
         defaultPreprocessors,
         defaultRubricPrompt,
+        config,
       );
       if (!parsedMembers || parsedMembers.length === 0) {
         logWarning(
@@ -623,6 +627,7 @@ async function parseGraderList(
         name,
         type: 'assert-set',
         assertions: parsedMembers,
+        ...(config !== undefined ? { config } : {}),
         ...(threshold !== undefined ? { threshold } : {}),
         ...(weight !== undefined ? { weight } : {}),
         ...(required !== undefined ? { required } : {}),
@@ -1675,6 +1680,26 @@ async function parseGraderList(
   return evaluators.length > 0 ? evaluators : undefined;
 }
 
+function withInheritedAssertionConfig(
+  rawEvaluator: JsonObject,
+  inheritedConfig?: JsonObject,
+): JsonObject {
+  const ownConfig = isJsonObject(rawEvaluator.config) ? rawEvaluator.config : undefined;
+  if (!inheritedConfig && !ownConfig) {
+    return rawEvaluator;
+  }
+
+  const mergedConfig = {
+    ...(inheritedConfig ?? {}),
+    ...(ownConfig ?? {}),
+  };
+
+  return {
+    ...rawEvaluator,
+    config: mergedConfig,
+  };
+}
+
 interface ParsedPromptField {
   readonly prompt?: string;
   readonly promptPath?: string;

packages/core/src/evaluation/types.ts

@@ -866,6 +866,7 @@ export type AssertSetGraderConfig = {
   readonly name: string;
   readonly type: 'assert-set';
   readonly assertions: readonly GraderConfig[];
+  readonly config?: JsonObject;
   readonly threshold?: number;
   readonly weight?: number;
   readonly required?: boolean;

packages/core/test/evaluation/graders/promptfoo-assertions.test.ts

@@ -194,6 +194,107 @@ describe('promptfoo-compatible built-in assertions', () => {
     expect(result.scores?.map((score) => score.type)).toEqual(['contains', 'starts-with']);
   });
 
+  it('without threshold follows child assertion pass/fail status', async () => {
+    const result = await run({
+      name: 'set',
+      type: 'assert-set',
+      assertions: [
+        { name: 'contains', type: 'contains', value: 'Paris' },
+        { name: 'missing', type: 'contains', value: 'Berlin' },
+      ],
+    });
+
+    expect(result.score).toBe(0.5);
+    expect(result.verdict).toBe('fail');
+    expect(result.details).toBeUndefined();
+  });
+
+  it('uses threshold to override child assertion pass/fail status', async () => {
+    const result = await run({
+      name: 'set',
+      type: 'assert-set',
+      threshold: 0.5,
+      assertions: [
+        { name: 'contains', type: 'contains', value: 'Paris' },
+        { name: 'missing', type: 'contains', value: 'Berlin' },
+      ],
+    });
+
+    expect(result.score).toBe(0.5);
+    expect(result.verdict).toBe('pass');
+    expect(result.details).toEqual({ threshold: 0.5 });
+  });
+
+  it('weights child assertion scores when aggregating assert-set results', async () => {
+    const result = await run({
+      name: 'weighted',
+      type: 'assert-set',
+      threshold: 0.6,
+      assertions: [
+        { name: 'contains', type: 'contains', value: 'Paris', weight: 2 },
+        { name: 'missing', type: 'contains', value: 'Berlin', weight: 1 },
+      ],
+    });
+
+    expect(result.score).toBeCloseTo(2 / 3);
+    expect(result.verdict).toBe('pass');
+    expect(result.scores?.map((score) => score.weight)).toEqual([2, 1]);
+  });
+
+  it('does not let zero-weight child failures fail no-threshold assert-sets', async () => {
+    const result = await run({
+      name: 'metric-only',
+      type: 'assert-set',
+      assertions: [
+        { name: 'contains', type: 'contains', value: 'Paris' },
+        { name: 'diagnostic', type: 'contains', value: 'Berlin', weight: 0 },
+      ],
+    });
+
+    expect(result.score).toBe(1);
+    expect(result.verdict).toBe('pass');
+  });
+
+  it('uses metric names for child score names and preserves nested assert-set scores', async () => {
+    const result = await run({
+      name: 'parent_metric',
+      type: 'assert-set',
+      threshold: 0.5,
+      assertions: [
+        { name: 'text_match', type: 'contains', value: 'Paris' },
+        {
+          name: 'nested_metric',
+          type: 'assert-set',
+          threshold: 1,
+          assertions: [{ name: 'starts_metric', type: 'starts-with', value: 'Paris' }],
+        },
+      ],
+    });
+
+    expect(result.scores?.map((score) => score.name)).toEqual(['text_match', 'nested_metric']);
+    expect(result.scores?.[1]?.scores?.map((score) => score.name)).toEqual(['starts_metric']);
+  });
+
+  it('passes assert-set config to script assertion context', async () => {
+    const result = await run({
+      name: 'configured',
+      type: 'assert-set',
+      config: {
+        expectedCity: 'Paris',
+      },
+      assertions: [
+        {
+          name: 'configured-js',
+          type: 'javascript',
+          value: 'context.config.expectedCity === "Paris"',
+        },
+      ],
+    });
+
+    expect(result.score).toBe(1);
+    expect(result.verdict).toBe('pass');
+  });
+
   it('does not count zero-score script children as passing in assert-set thresholds', async () => {
     const result = await run({
       name: 'gate',