From 30794787f122dae8f330042b327da98eb4536117 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Fri, 21 Aug 2026 16:36:48 +0000 Subject: [PATCH] fix(evaluator): include user_message placeholder for TOOL_CALL evaluators The Tool-level (TOOL_CALL) prompt-template allowlist omitted the user_message skill placeholder that AgentCore Evaluations supports, so the add-evaluator wizard and the validation error text never surfaced it. Add it to LEVEL_PLACEHOLDERS.TOOL_CALL and PLACEHOLDER_DESCRIPTIONS, and assert it in the placeholder test. Ref: https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/prompt-templates-builtin.html --- src/cli/tui/screens/evaluator/__tests__/types.test.ts | 1 + src/cli/tui/screens/evaluator/types.ts | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/cli/tui/screens/evaluator/__tests__/types.test.ts b/src/cli/tui/screens/evaluator/__tests__/types.test.ts index 4f3267549..6899fb981 100644 --- a/src/cli/tui/screens/evaluator/__tests__/types.test.ts +++ b/src/cli/tui/screens/evaluator/__tests__/types.test.ts @@ -39,6 +39,7 @@ describe('LEVEL_PLACEHOLDERS', () => { expect(LEVEL_PLACEHOLDERS.TOOL_CALL).toContain('available_skills'); expect(LEVEL_PLACEHOLDERS.TOOL_CALL).toContain('invoked_skill'); expect(LEVEL_PLACEHOLDERS.TOOL_CALL).toContain('skill_content'); + expect(LEVEL_PLACEHOLDERS.TOOL_CALL).toContain('user_message'); }); }); diff --git a/src/cli/tui/screens/evaluator/types.ts b/src/cli/tui/screens/evaluator/types.ts index f929ccc25..46e45986a 100644 --- a/src/cli/tui/screens/evaluator/types.ts +++ b/src/cli/tui/screens/evaluator/types.ts @@ -195,7 +195,15 @@ export function getEvaluatorModelOptions(provider: EvaluatorModelProvider): Eval export const LEVEL_PLACEHOLDERS: Record = { SESSION: ['context', 'available_tools'], TRACE: ['context', 'assistant_turn'], - TOOL_CALL: ['available_tools', 'context', 'tool_turn', 'available_skills', 'invoked_skill', 'skill_content'], + TOOL_CALL: [ + 'available_tools', + 'context', + 'tool_turn', + 'available_skills', + 'invoked_skill', + 'skill_content', + 'user_message', + ], }; /** @@ -221,6 +229,7 @@ export const PLACEHOLDER_DESCRIPTIONS: Record = { available_skills: 'skills offered to the agent (name + description)', invoked_skill: 'the skill the agent selected for this span', skill_content: "the selected skill's SKILL.md instructions", + user_message: 'the user request in the turn that triggered the skill invocation', }; /**