From 2abf14c223bd71c0343e1589e0b140132e9a0ce2 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Mon, 17 Aug 2026 19:38:03 +0000 Subject: [PATCH] feat(evaluator): surface skill evaluators (TOOL_CALL level + skill placeholders) - run-eval: map Builtin.SkillSelectionAccuracy / Builtin.SkillInstructionFollowing to TOOL_CALL so they target skill-invocation spans instead of falling through to the SESSION default (which scores the wrong spans, silently). - evaluator TUI: expose {available_skills} / {invoked_skill} / {skill_content} placeholders for TOOL_CALL custom LLM-judge evaluators, matching the service prompt-template contract. - tests for both. Follow-up (not in this PR): replace the static BUILTIN_EVALUATOR_LEVELS map with a cached listEvaluators() lookup so future built-ins self-resolve. --- npm-shrinkwrap.json | 4 +-- .../eval/__tests__/run-eval.test.ts | 36 +++++++++++++++++++ src/cli/operations/eval/run-eval.ts | 4 +++ .../screens/evaluator/__tests__/types.test.ts | 6 ++++ src/cli/tui/screens/evaluator/types.ts | 5 ++- 5 files changed, 52 insertions(+), 3 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 51381483d..559d62d97 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,12 +1,12 @@ { "name": "@aws/agentcore", - "version": "0.26.0", + "version": "0.27.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@aws/agentcore", - "version": "0.26.0", + "version": "0.27.0", "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { diff --git a/src/cli/operations/eval/__tests__/run-eval.test.ts b/src/cli/operations/eval/__tests__/run-eval.test.ts index d862ab9a0..91ab6a92a 100644 --- a/src/cli/operations/eval/__tests__/run-eval.test.ts +++ b/src/cli/operations/eval/__tests__/run-eval.test.ts @@ -847,6 +847,42 @@ describe('handleRunEval', () => { ); }); + it('resolves Builtin skill evaluators to TOOL_CALL level', async () => { + const ctx = makeDeployedContext(); + mockLoadDeployedProjectConfig.mockResolvedValue(ctx); + mockResolveAgent.mockReturnValue({ + success: true, + agent: { + agentName: 'my-agent', + targetName: 'dev', + region: 'us-east-1', + accountId: '111222333444', + runtimeId: 'rt-123', + }, + }); + + const spanRows = [makeToolCallSpanRow('session-1', 'trace-1', 'span-tool-1', 'calculator')]; + setupCloudWatchToReturn(spanRows); + + mockEvaluate.mockResolvedValue({ + evaluationResults: [{ value: 1.0, context: { spanContext: { sessionId: 'session-1', spanId: 'span-tool-1' } } }], + }); + + // Builtin.SkillSelectionAccuracy / SkillInstructionFollowing are TOOL_CALL-level — they must + // target spans, not default to SESSION (which would send no targetSpanIds). + const result = await handleRunEval({ + evaluator: ['Builtin.SkillSelectionAccuracy', 'Builtin.SkillInstructionFollowing'], + days: 7, + }); + + expect(result.success).toBe(true); + expect(mockEvaluate).toHaveBeenCalledWith( + expect.objectContaining({ + targetSpanIds: ['span-tool-1'], + }) + ); + }); + it('batches targetSpanIds into chunks of 10 for TOOL_CALL evaluators', async () => { const ctx = makeDeployedContext(); mockLoadDeployedProjectConfig.mockResolvedValue(ctx); diff --git a/src/cli/operations/eval/run-eval.ts b/src/cli/operations/eval/run-eval.ts index 45f35586b..e9dcd37ae 100644 --- a/src/cli/operations/eval/run-eval.ts +++ b/src/cli/operations/eval/run-eval.ts @@ -179,6 +179,10 @@ const BUILTIN_EVALUATOR_LEVELS: Record = { 'Builtin.InstructionFollowing': 'TRACE', 'Builtin.Refusal': 'TRACE', 'Builtin.ToolSelectionAccuracy': 'TOOL_CALL', + // Skill evaluators judge the span that carries a skill invocation, so they are TOOL_CALL — + // not the SESSION default an unmapped Builtin.* falls through to, which would score wrong spans. + 'Builtin.SkillSelectionAccuracy': 'TOOL_CALL', + 'Builtin.SkillInstructionFollowing': 'TOOL_CALL', }; /** diff --git a/src/cli/tui/screens/evaluator/__tests__/types.test.ts b/src/cli/tui/screens/evaluator/__tests__/types.test.ts index 00612c205..4f3267549 100644 --- a/src/cli/tui/screens/evaluator/__tests__/types.test.ts +++ b/src/cli/tui/screens/evaluator/__tests__/types.test.ts @@ -34,6 +34,12 @@ describe('LEVEL_PLACEHOLDERS', () => { expect(LEVEL_PLACEHOLDERS.TOOL_CALL).toContain('context'); expect(LEVEL_PLACEHOLDERS.TOOL_CALL).toContain('tool_turn'); }); + + it('TOOL_CALL exposes skill placeholders for skill evaluators', () => { + expect(LEVEL_PLACEHOLDERS.TOOL_CALL).toContain('available_skills'); + expect(LEVEL_PLACEHOLDERS.TOOL_CALL).toContain('invoked_skill'); + expect(LEVEL_PLACEHOLDERS.TOOL_CALL).toContain('skill_content'); + }); }); describe('DEFAULT_INSTRUCTIONS', () => { diff --git a/src/cli/tui/screens/evaluator/types.ts b/src/cli/tui/screens/evaluator/types.ts index 00101ac35..f929ccc25 100644 --- a/src/cli/tui/screens/evaluator/types.ts +++ b/src/cli/tui/screens/evaluator/types.ts @@ -195,7 +195,7 @@ export function getEvaluatorModelOptions(provider: EvaluatorModelProvider): Eval export const LEVEL_PLACEHOLDERS: Record = { SESSION: ['context', 'available_tools'], TRACE: ['context', 'assistant_turn'], - TOOL_CALL: ['available_tools', 'context', 'tool_turn'], + TOOL_CALL: ['available_tools', 'context', 'tool_turn', 'available_skills', 'invoked_skill', 'skill_content'], }; /** @@ -218,6 +218,9 @@ export const PLACEHOLDER_DESCRIPTIONS: Record = { expected_tool_trajectory: 'caller-provided expected sequence of tool calls', actual_tool_trajectory: 'actual sequence of tool calls from the session', expected_response: 'caller-provided expected agent response', + available_skills: 'skills offered to the agent (name + description)', + invoked_skill: 'the skill the agent selected for this span', + skill_content: "the selected skill's SKILL.md instructions", }; /**