From 8ff265749c91226e6ca5b655d9406bd6b52f1c43 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Sat, 22 Aug 2026 15:51:47 +0000 Subject: [PATCH 1/2] =?UTF-8?q?feat(eval):=20add=20`ondemand=20simulate`?= =?UTF-8?q?=20=E2=80=94=20replay=20a=20dataset,=20evaluate=20synchronously?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On-demand twin of batch-evaluation simulate. Handler composes three existing core calls — invokeDataset (replay) -> getTracesForAgent (gather) -> evaluate (sync grade) — and adapts each session's neutral InlineGroundTruth into EvaluationReferenceInput[] (assertions + expectedTrajectory map 1:1, correlated by sessionId). No new core method. v1 = session-level ground truth only; per-turn expectedResponse is trace-level and needs a turn->trace id we don't have client-side, so it is omitted (batch simulate still covers it). --- src/handlers/eval/ondemand/index.tsx | 4 +- src/handlers/eval/ondemand/ondemand.test.tsx | 4 +- .../__snapshots__/simulate.test.tsx.snap | 23 +++ src/handlers/eval/ondemand/simulate/index.tsx | 123 ++++++++++++ .../eval/ondemand/simulate/simulate.test.tsx | 187 ++++++++++++++++++ 5 files changed, 338 insertions(+), 3 deletions(-) create mode 100644 src/handlers/eval/ondemand/simulate/__snapshots__/simulate.test.tsx.snap create mode 100644 src/handlers/eval/ondemand/simulate/index.tsx create mode 100644 src/handlers/eval/ondemand/simulate/simulate.test.tsx diff --git a/src/handlers/eval/ondemand/index.tsx b/src/handlers/eval/ondemand/index.tsx index d46cc38ea..e731df263 100644 --- a/src/handlers/eval/ondemand/index.tsx +++ b/src/handlers/eval/ondemand/index.tsx @@ -3,11 +3,13 @@ import type { AppIO } from "../../../io"; import type { Core } from "../../types"; import { createHelpDefault } from "../../help"; import { createEvaluateOnDemandHandler } from "./evaluate"; +import { createSimulateOnDemandHandler } from "./simulate"; // ondemand groups the synchronous, client-side evaluation commands. It has no TUI // screen (unlike evaluator/online-eval), so a bare invocation prints help. export function createOnDemandHandler(core: Core, io: AppIO): Router { return new Router("ondemand", "evaluate existing sessions synchronously, client-side") .default(createHelpDefault(io)) - .handler(createEvaluateOnDemandHandler(core, io)); + .handler(createEvaluateOnDemandHandler(core, io)) + .handler(createSimulateOnDemandHandler(core, io)); } diff --git a/src/handlers/eval/ondemand/ondemand.test.tsx b/src/handlers/eval/ondemand/ondemand.test.tsx index 903a9140e..5bbb55ca2 100644 --- a/src/handlers/eval/ondemand/ondemand.test.tsx +++ b/src/handlers/eval/ondemand/ondemand.test.tsx @@ -168,7 +168,7 @@ const BASE = [ ]; describe("eval ondemand command hierarchy", () => { - test("registers evaluate under eval → ondemand", () => { + test("registers evaluate + simulate under eval → ondemand", () => { const io = testIO(); const root = createRootHandler(new TestCoreClient(), { io: io.io, @@ -180,7 +180,7 @@ describe("eval ondemand command hierarchy", () => { .find((c) => c.name() === "eval") ?.children() .find((c) => c.name() === "ondemand"); - expect(group?.children().map((c) => c.name())).toEqual(["evaluate"]); + expect(group?.children().map((c) => c.name())).toEqual(["evaluate", "simulate"]); }); }); diff --git a/src/handlers/eval/ondemand/simulate/__snapshots__/simulate.test.tsx.snap b/src/handlers/eval/ondemand/simulate/__snapshots__/simulate.test.tsx.snap new file mode 100644 index 000000000..ff879c224 --- /dev/null +++ b/src/handlers/eval/ondemand/simulate/__snapshots__/simulate.test.tsx.snap @@ -0,0 +1,23 @@ +// Bun Snapshot v1, https://bun.sh/docs/test/snapshots + +exports[`eval ondemand simulate adapts inline ground truth to EvaluationReferenceInput [golden] 1`] = ` +[ + { + "assertions": [ + { + "text": "polite", + }, + ], + "context": { + "spanContext": { + "sessionId": "s1", + }, + }, + "expectedTrajectory": { + "toolNames": [ + "lookup", + ], + }, + }, +] +`; diff --git a/src/handlers/eval/ondemand/simulate/index.tsx b/src/handlers/eval/ondemand/simulate/index.tsx new file mode 100644 index 000000000..322b44eea --- /dev/null +++ b/src/handlers/eval/ondemand/simulate/index.tsx @@ -0,0 +1,123 @@ +import z from "zod"; +import type { EvaluationReferenceInput } from "@aws-sdk/client-bedrock-agentcore"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import type { InvokedSession } from "../../types"; +import { coreOptsFromCtx } from "../../../utils"; +import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request"; + +// Composes invokeDataset (replay) → getTracesForAgent (gather) → evaluate (grade, +// synchronous). The on-demand twin of batch-evaluation simulate: no async job, scores +// print inline. Invoke flags mirror `runtime invoke`. +export const createSimulateOnDemandHandler = (core: Core, _io: AppIO) => + createHandler({ + name: "simulate", + description: "replay a dataset against a runtime, then evaluate the sessions client-side", + flags: [ + flag("runtime-id", "runtime id to invoke per scenario", z.string().optional()), + flag("qualifier", "runtime endpoint qualifier (default DEFAULT)", z.string().optional()), + flag( + "payload-template", + 'JSON payload template; {input} is the scenario input, e.g. {"prompt":"{input}"}', + z.string().optional(), + ), + flag("header", "an ordered application header (repeatable)", z.array(z.string()).optional()), + flag( + "bearer-token", + "CUSTOM_JWT bearer token (for JWT-auth runtimes)", + z.string().optional(), + ), + flag("user-id", "runtime user id", z.string().optional()), + flag("dataset", "dataset source: local JSONL path or a dataset id", z.string().optional()), + flag("dataset-version", "dataset version (with a dataset id)", z.string().optional()), + flag("evaluator", "evaluator id(s) to apply", z.array(z.string()).optional()), + ], + handle: async (ctx, flags) => { + if (!flags["runtime-id"]) + throw new InputValidationError("required option '--runtime-id' not specified"); + if (!flags["payload-template"]) { + throw new InputValidationError("required option '--payload-template' not specified"); + } + if (!flags["dataset"]) + throw new InputValidationError("required option '--dataset' not specified"); + if (!flags["evaluator"]?.length) { + throw new InputValidationError( + "required option '--evaluator ' not specified", + ); + } + + // Ctrl-C aborts the run (invokes, the ingestion wait, the dataset download). + const controller = new AbortController(); + const interrupt = () => controller.abort(); + process.once("SIGINT", interrupt); + try { + const opts = coreOptsFromCtx(ctx); + + // 1. Replay the dataset — reuse invokeDataset verbatim (grader-agnostic). + const replay = await core.eval.invokeDataset( + { + runtimeId: flags["runtime-id"], + qualifier: flags["qualifier"], + payloadTemplate: flags["payload-template"], + headers: parseRuntimeInvokeHeaders(flags["header"]), + bearerToken: flags["bearer-token"], + userId: flags["user-id"], + dataset: flags["dataset"], + datasetVersion: flags["dataset-version"], + }, + opts, + controller.signal, + ); + if (replay.invoked === 0) { + const detail = replay.firstError ? `; first error: ${replay.firstError.message}` : ""; + throw new InputValidationError( + `no examples could be invoked (${replay.failed} failed) — nothing to evaluate${detail}`, + ); + } + + // 2. Gather the just-created sessions' traces (client-side CloudWatch read). + const traces = await core.eval.getTracesForAgent( + { + agent: flags["runtime-id"], + endpoint: flags["qualifier"], + sessionIds: replay.sessions.map((s) => s.sessionId), + }, + opts, + ); + + // 3. Adapt neutral ground truth → EvaluationReferenceInput[] and grade synchronously. + const groundTruth = replay.sessions.flatMap(toReferenceInputs); + const result = await core.eval.evaluate( + { traces, evaluatorIds: flags["evaluator"], groundTruth }, + opts, + ); + + ctx.require(JsonRendererKey).renderJson({ + ...result, + examplesInvoked: replay.invoked, + examplesFailed: replay.failed, + }); + } finally { + process.off("SIGINT", interrupt); + } + }, + }); + +// Adapt one invoked session's neutral InlineGroundTruth to the Evaluate API's +// EvaluationReferenceInput, correlated by sessionId. assertions ({text}[]) and +// expectedTrajectory ({toolNames}) map 1:1. Per-turn expectedResponse is trace-level and +// needs a turn→trace id we don't have here, so it is omitted (batch simulate covers it). +function toReferenceInputs(s: InvokedSession): EvaluationReferenceInput[] { + const gt = s.groundTruth; + if (!gt?.assertions?.length && !gt?.expectedTrajectory) return []; + return [ + { + context: { spanContext: { sessionId: s.sessionId } }, + ...(gt.assertions?.length && { assertions: gt.assertions }), + ...(gt.expectedTrajectory && { expectedTrajectory: gt.expectedTrajectory }), + }, + ]; +} diff --git a/src/handlers/eval/ondemand/simulate/simulate.test.tsx b/src/handlers/eval/ondemand/simulate/simulate.test.tsx new file mode 100644 index 000000000..57bfd9595 --- /dev/null +++ b/src/handlers/eval/ondemand/simulate/simulate.test.tsx @@ -0,0 +1,187 @@ +import { test, expect, describe } from "bun:test"; +import { createRootHandler } from "../../../index"; +import { + createSilentLogger, + TestCoreClient, + testIO, + TestGlobalConfigAccessor, +} from "../../../../testing"; +import type { EvaluateResult, InvokeDatasetResult, SessionTrace } from "../../types"; + +// Two invoked sessions: e1 carries session-level ground truth (assertions + trajectory), +// e2 carries none. The handler adapts these to EvaluationReferenceInput[] for evaluate. +const INVOKE_RESULT: InvokeDatasetResult = { + sessions: [ + { + exampleId: "e1", + sessionId: "s1", + groundTruth: { + assertions: [{ text: "polite" }], + expectedTrajectory: { toolNames: ["lookup"] }, + // turn-level expectedResponse is intentionally present to prove it is dropped (v1). + turns: [{ input: { prompt: "hi" }, expectedResponse: { text: "hello" } }], + }, + }, + { exampleId: "e2", sessionId: "s2" }, + ], + invoked: 2, + failed: 0, +}; + +const TRACES: SessionTrace[] = [ + { sessionId: "s1", spans: [], traceIds: ["t1"], toolCallSpanIds: [] }, + { sessionId: "s2", spans: [], traceIds: ["t2"], toolCallSpanIds: [] }, +]; + +const EVAL_RESULT: EvaluateResult = { sessionsRequested: 2, sessionsEvaluated: 2, results: [] }; + +async function run(args: string[], configure?: (core: TestCoreClient) => void) { + const core = new TestCoreClient(); + core.eval.setInvokeDatasetResponse(INVOKE_RESULT); + core.eval.setGetTracesResponse(TRACES); + core.eval.setEvaluateResponse(EVAL_RESULT); + configure?.(core); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); + return { core, stdout: io.stdout() }; +} + +const BASE = [ + "eval", + "ondemand", + "simulate", + "--runtime-id", + "r-1", + "--payload-template", + '{"prompt":"{input}"}', + "--dataset", + "/tmp/ds.jsonl", + "--evaluator", + "Builtin.Helpfulness", +]; + +describe("eval ondemand simulate", () => { + test("registered under ondemand", () => { + const io = testIO(); + const root = createRootHandler(new TestCoreClient(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + // Bare `ondemand` prints help listing both subcommands. + expect(root).toBeDefined(); + }); + + test.each([ + [ + [ + "eval", + "ondemand", + "simulate", + "--payload-template", + "{}", + "--dataset", + "d", + "--evaluator", + "E", + ], + /--runtime-id/, + ], + [ + ["eval", "ondemand", "simulate", "--runtime-id", "r", "--dataset", "d", "--evaluator", "E"], + /--payload-template/, + ], + [ + [ + "eval", + "ondemand", + "simulate", + "--runtime-id", + "r", + "--payload-template", + "{}", + "--evaluator", + "E", + ], + /--dataset/, + ], + [ + [ + "eval", + "ondemand", + "simulate", + "--runtime-id", + "r", + "--payload-template", + "{}", + "--dataset", + "d", + ], + /--evaluator/, + ], + ])("rejects missing required flag", async (args, expected) => { + await expect(run(args)).rejects.toThrow(expected); + }); + + test("composes invokeDataset → getTracesForAgent → evaluate", async () => { + const { core } = await run([...BASE, "--qualifier", "PROD", "--header", "x-a:1"]); + + const invoke = core.eval.calls.find((c) => c.method === "invokeDataset"); + expect(invoke?.args[0]).toMatchObject({ + runtimeId: "r-1", + qualifier: "PROD", + payloadTemplate: '{"prompt":"{input}"}', + headers: [["x-a", "1"]], + dataset: "/tmp/ds.jsonl", + }); + // Ctrl-C signal threaded to the replay. + expect(invoke?.args[2]).toBeInstanceOf(AbortSignal); + + // getTracesForAgent is asked for exactly the sessions the replay created. + const traces = core.eval.calls.find((c) => c.method === "getTracesForAgent"); + expect(traces?.args[0]).toMatchObject({ + agent: "r-1", + endpoint: "PROD", + sessionIds: ["s1", "s2"], + }); + + // evaluate gets the runtime evaluators over the gathered traces. + const evaluate = core.eval.calls.find((c) => c.method === "evaluate"); + expect(evaluate?.args[0]).toMatchObject({ evaluatorIds: ["Builtin.Helpfulness"] }); + }); + + // Golden: the adapted EvaluationReferenceInput[] handed to evaluate. Locks session-level + // assertions/trajectory (1:1), the sessionId correlation, and that e2 (no GT) and the + // turn-level expectedResponse both contribute nothing. + test("adapts inline ground truth to EvaluationReferenceInput [golden]", async () => { + const { core } = await run(BASE); + const evaluate = core.eval.calls.find((c) => c.method === "evaluate"); + expect(evaluate).toBeDefined(); + const input = evaluate!.args[0] as { groundTruth: unknown }; + expect(input.groundTruth).toMatchSnapshot(); + }); + + test("renders scores inline with invoked/failed counts", async () => { + const { stdout } = await run(BASE); + expect(JSON.parse(stdout)).toEqual({ + sessionsRequested: 2, + sessionsEvaluated: 2, + results: [], + examplesInvoked: 2, + examplesFailed: 0, + }); + }); + + test("refuses to evaluate when nothing was invoked", async () => { + await expect( + run(BASE, (core) => + core.eval.setInvokeDatasetResponse({ sessions: [], invoked: 0, failed: 3 }), + ), + ).rejects.toThrow(/no examples could be invoked \(3 failed\)/); + }); +}); From 94a16ac2e840c2755d6780f7e696c2199043b6e2 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Sat, 22 Aug 2026 16:11:34 +0000 Subject: [PATCH 2/2] test(eval): ondemand simulate uses matchGolden committed goldens (runtime-handler pattern) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch the golden assertions from bun toMatchSnapshot to matchGolden + committed __fixtures__/*.golden.json, matching the runtime handlers' golden style. Driven by TestCoreClient: fixtureFactories record/replay can't key this command because its inputs carry random per-session UUIDs + now-based CloudWatch windows (fixturePath hashes the input), so replay never matches — the same reason batch simulate uses TestCoreClient. --- .../ondemand-simulate-groundtruth.golden.json | 19 +++++++++++++++ .../ondemand-simulate-output.golden.json | 7 ++++++ .../__snapshots__/simulate.test.tsx.snap | 23 ------------------- .../eval/ondemand/simulate/simulate.test.tsx | 20 ++++++++-------- 4 files changed, 37 insertions(+), 32 deletions(-) create mode 100644 src/handlers/eval/ondemand/simulate/__fixtures__/ondemand-simulate-groundtruth.golden.json create mode 100644 src/handlers/eval/ondemand/simulate/__fixtures__/ondemand-simulate-output.golden.json delete mode 100644 src/handlers/eval/ondemand/simulate/__snapshots__/simulate.test.tsx.snap diff --git a/src/handlers/eval/ondemand/simulate/__fixtures__/ondemand-simulate-groundtruth.golden.json b/src/handlers/eval/ondemand/simulate/__fixtures__/ondemand-simulate-groundtruth.golden.json new file mode 100644 index 000000000..81d7bfbba --- /dev/null +++ b/src/handlers/eval/ondemand/simulate/__fixtures__/ondemand-simulate-groundtruth.golden.json @@ -0,0 +1,19 @@ +[ + { + "context": { + "spanContext": { + "sessionId": "s1" + } + }, + "assertions": [ + { + "text": "polite" + } + ], + "expectedTrajectory": { + "toolNames": [ + "lookup" + ] + } + } +] \ No newline at end of file diff --git a/src/handlers/eval/ondemand/simulate/__fixtures__/ondemand-simulate-output.golden.json b/src/handlers/eval/ondemand/simulate/__fixtures__/ondemand-simulate-output.golden.json new file mode 100644 index 000000000..21b06350a --- /dev/null +++ b/src/handlers/eval/ondemand/simulate/__fixtures__/ondemand-simulate-output.golden.json @@ -0,0 +1,7 @@ +{ + "sessionsRequested": 2, + "sessionsEvaluated": 2, + "results": [], + "examplesInvoked": 2, + "examplesFailed": 0 +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/simulate/__snapshots__/simulate.test.tsx.snap b/src/handlers/eval/ondemand/simulate/__snapshots__/simulate.test.tsx.snap deleted file mode 100644 index ff879c224..000000000 --- a/src/handlers/eval/ondemand/simulate/__snapshots__/simulate.test.tsx.snap +++ /dev/null @@ -1,23 +0,0 @@ -// Bun Snapshot v1, https://bun.sh/docs/test/snapshots - -exports[`eval ondemand simulate adapts inline ground truth to EvaluationReferenceInput [golden] 1`] = ` -[ - { - "assertions": [ - { - "text": "polite", - }, - ], - "context": { - "spanContext": { - "sessionId": "s1", - }, - }, - "expectedTrajectory": { - "toolNames": [ - "lookup", - ], - }, - }, -] -`; diff --git a/src/handlers/eval/ondemand/simulate/simulate.test.tsx b/src/handlers/eval/ondemand/simulate/simulate.test.tsx index 57bfd9595..8d04a162b 100644 --- a/src/handlers/eval/ondemand/simulate/simulate.test.tsx +++ b/src/handlers/eval/ondemand/simulate/simulate.test.tsx @@ -1,13 +1,17 @@ import { test, expect, describe } from "bun:test"; +import { join } from "node:path"; import { createRootHandler } from "../../../index"; import { createSilentLogger, + matchGolden, TestCoreClient, testIO, TestGlobalConfigAccessor, } from "../../../../testing"; import type { EvaluateResult, InvokeDatasetResult, SessionTrace } from "../../types"; +const FIXTURES = join(import.meta.dir, "__fixtures__"); + // Two invoked sessions: e1 carries session-level ground truth (assertions + trajectory), // e2 carries none. The handler adapts these to EvaluationReferenceInput[] for evaluate. const INVOKE_RESULT: InvokeDatasetResult = { @@ -163,18 +167,16 @@ describe("eval ondemand simulate", () => { const evaluate = core.eval.calls.find((c) => c.method === "evaluate"); expect(evaluate).toBeDefined(); const input = evaluate!.args[0] as { groundTruth: unknown }; - expect(input.groundTruth).toMatchSnapshot(); + matchGolden( + FIXTURES, + "ondemand-simulate-groundtruth.golden.json", + JSON.stringify(input.groundTruth, null, 2), + ); }); - test("renders scores inline with invoked/failed counts", async () => { + test("renders scores inline with invoked/failed counts [golden]", async () => { const { stdout } = await run(BASE); - expect(JSON.parse(stdout)).toEqual({ - sessionsRequested: 2, - sessionsEvaluated: 2, - results: [], - examplesInvoked: 2, - examplesFailed: 0, - }); + matchGolden(FIXTURES, "ondemand-simulate-output.golden.json", stdout); }); test("refuses to evaluate when nothing was invoked", async () => {