feat(eval): ondemand simulate — replay a dataset, evaluate synchronously

src/handlers/eval/ondemand/index.tsx

@@ -3,11 +3,13 @@ import type { AppIO } from "../../../io";
 import type { Core } from "../../types";
 import { createHelpDefault } from "../../help";
 import { createEvaluateOnDemandHandler } from "./evaluate";
+import { createSimulateOnDemandHandler } from "./simulate";
 
 // ondemand groups the synchronous, client-side evaluation commands. It has no TUI
 // screen (unlike evaluator/online-eval), so a bare invocation prints help.
 export function createOnDemandHandler(core: Core, io: AppIO): Router {
   return new Router("ondemand", "evaluate existing sessions synchronously, client-side")
     .default(createHelpDefault(io))
-    .handler(createEvaluateOnDemandHandler(core, io));
+    .handler(createEvaluateOnDemandHandler(core, io))
+    .handler(createSimulateOnDemandHandler(core, io));
 }

src/handlers/eval/ondemand/ondemand.test.tsx

@@ -168,7 +168,7 @@ const BASE = [
 ];
 
 describe("eval ondemand command hierarchy", () => {
-  test("registers evaluate under eval → ondemand", () => {
+  test("registers evaluate + simulate under eval → ondemand", () => {
     const io = testIO();
     const root = createRootHandler(new TestCoreClient(), {
       io: io.io,
@@ -180,7 +180,7 @@ describe("eval ondemand command hierarchy", () => {
       .find((c) => c.name() === "eval")
       ?.children()
       .find((c) => c.name() === "ondemand");
-    expect(group?.children().map((c) => c.name())).toEqual(["evaluate"]);
+    expect(group?.children().map((c) => c.name())).toEqual(["evaluate", "simulate"]);
   });
 });
 

src/handlers/eval/ondemand/simulate/__fixtures__/ondemand-simulate-groundtruth.golden.json

@@ -0,0 +1,19 @@
+[
+  {
+    "context": {
+      "spanContext": {
+        "sessionId": "s1"
+      }
+    },
+    "assertions": [
+      {
+        "text": "polite"
+      }
+    ],
+    "expectedTrajectory": {
+      "toolNames": [
+        "lookup"
+      ]
+    }
+  }
+]

src/handlers/eval/ondemand/simulate/__fixtures__/ondemand-simulate-output.golden.json

@@ -0,0 +1,7 @@
+{
+  "sessionsRequested": 2,
+  "sessionsEvaluated": 2,
+  "results": [],
+  "examplesInvoked": 2,
+  "examplesFailed": 0
+}

src/handlers/eval/ondemand/simulate/index.tsx

@@ -0,0 +1,123 @@
+import z from "zod";
+import type { EvaluationReferenceInput } from "@aws-sdk/client-bedrock-agentcore";
+import { createHandler, flag } from "../../../../router";
+import { InputValidationError } from "../../../../errors";
+import { JsonRendererKey } from "../../../../tui";
+import type { AppIO } from "../../../../io";
+import type { Core } from "../../../types";
+import type { InvokedSession } from "../../types";
+import { coreOptsFromCtx } from "../../../utils";
+import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request";
+
+// Composes invokeDataset (replay) → getTracesForAgent (gather) → evaluate (grade,
+// synchronous). The on-demand twin of batch-evaluation simulate: no async job, scores
+// print inline. Invoke flags mirror `runtime invoke`.
+export const createSimulateOnDemandHandler = (core: Core, _io: AppIO) =>
+  createHandler({
+    name: "simulate",
+    description: "replay a dataset against a runtime, then evaluate the sessions client-side",
+    flags: [
+      flag("runtime-id", "runtime id to invoke per scenario", z.string().optional()),
+      flag("qualifier", "runtime endpoint qualifier (default DEFAULT)", z.string().optional()),
+      flag(
+        "payload-template",
+        'JSON payload template; {input} is the scenario input, e.g. {"prompt":"{input}"}',
+        z.string().optional(),
+      ),
+      flag("header", "an ordered application header (repeatable)", z.array(z.string()).optional()),
+      flag(
+        "bearer-token",
+        "CUSTOM_JWT bearer token (for JWT-auth runtimes)",
+        z.string().optional(),
+      ),
+      flag("user-id", "runtime user id", z.string().optional()),
+      flag("dataset", "dataset source: local JSONL path or a dataset id", z.string().optional()),
+      flag("dataset-version", "dataset version (with a dataset id)", z.string().optional()),
+      flag("evaluator", "evaluator id(s) to apply", z.array(z.string()).optional()),
+    ],
+    handle: async (ctx, flags) => {
+      if (!flags["runtime-id"])
+        throw new InputValidationError("required option '--runtime-id' not specified");
+      if (!flags["payload-template"]) {
+        throw new InputValidationError("required option '--payload-template' not specified");
+      }
+      if (!flags["dataset"])
+        throw new InputValidationError("required option '--dataset' not specified");
+      if (!flags["evaluator"]?.length) {
+        throw new InputValidationError(
+          "required option '--evaluator <evaluator...>' not specified",
+        );
+      }
+
+      // Ctrl-C aborts the run (invokes, the ingestion wait, the dataset download).
+      const controller = new AbortController();
+      const interrupt = () => controller.abort();
+      process.once("SIGINT", interrupt);
+      try {
+        const opts = coreOptsFromCtx(ctx);
+
+        // 1. Replay the dataset — reuse invokeDataset verbatim (grader-agnostic).
+        const replay = await core.eval.invokeDataset(
+          {
+            runtimeId: flags["runtime-id"],
+            qualifier: flags["qualifier"],
+            payloadTemplate: flags["payload-template"],
+            headers: parseRuntimeInvokeHeaders(flags["header"]),
+            bearerToken: flags["bearer-token"],
+            userId: flags["user-id"],
+            dataset: flags["dataset"],
+            datasetVersion: flags["dataset-version"],
+          },
+          opts,
+          controller.signal,
+        );
+        if (replay.invoked === 0) {
+          const detail = replay.firstError ? `; first error: ${replay.firstError.message}` : "";
+          throw new InputValidationError(
+            `no examples could be invoked (${replay.failed} failed) — nothing to evaluate${detail}`,
+          );
+        }
+
+        // 2. Gather the just-created sessions' traces (client-side CloudWatch read).
+        const traces = await core.eval.getTracesForAgent(
+          {
+            agent: flags["runtime-id"],
+            endpoint: flags["qualifier"],
+            sessionIds: replay.sessions.map((s) => s.sessionId),
+          },
+          opts,
+        );
+
+        // 3. Adapt neutral ground truth → EvaluationReferenceInput[] and grade synchronously.
+        const groundTruth = replay.sessions.flatMap(toReferenceInputs);
+        const result = await core.eval.evaluate(
+          { traces, evaluatorIds: flags["evaluator"], groundTruth },
+          opts,
+        );
+
+        ctx.require(JsonRendererKey).renderJson({
+          ...result,
+          examplesInvoked: replay.invoked,
+          examplesFailed: replay.failed,
+        });
+      } finally {
+        process.off("SIGINT", interrupt);
+      }
+    },
+  });
+
+// Adapt one invoked session's neutral InlineGroundTruth to the Evaluate API's
+// EvaluationReferenceInput, correlated by sessionId. assertions ({text}[]) and
+// expectedTrajectory ({toolNames}) map 1:1. Per-turn expectedResponse is trace-level and
+// needs a turn→trace id we don't have here, so it is omitted (batch simulate covers it).
+function toReferenceInputs(s: InvokedSession): EvaluationReferenceInput[] {
+  const gt = s.groundTruth;
+  if (!gt?.assertions?.length && !gt?.expectedTrajectory) return [];
+  return [
+    {
+      context: { spanContext: { sessionId: s.sessionId } },
+      ...(gt.assertions?.length && { assertions: gt.assertions }),
+      ...(gt.expectedTrajectory && { expectedTrajectory: gt.expectedTrajectory }),
+    },
+  ];
+}

src/handlers/eval/ondemand/simulate/simulate.test.tsx

@@ -0,0 +1,189 @@
+import { test, expect, describe } from "bun:test";
+import { join } from "node:path";
+import { createRootHandler } from "../../../index";
+import {
+  createSilentLogger,
+  matchGolden,
+  TestCoreClient,
+  testIO,
+  TestGlobalConfigAccessor,
+} from "../../../../testing";
+import type { EvaluateResult, InvokeDatasetResult, SessionTrace } from "../../types";
+
+const FIXTURES = join(import.meta.dir, "__fixtures__");
+
+// Two invoked sessions: e1 carries session-level ground truth (assertions + trajectory),
+// e2 carries none. The handler adapts these to EvaluationReferenceInput[] for evaluate.
+const INVOKE_RESULT: InvokeDatasetResult = {
+  sessions: [
+    {
+      exampleId: "e1",
+      sessionId: "s1",
+      groundTruth: {
+        assertions: [{ text: "polite" }],
+        expectedTrajectory: { toolNames: ["lookup"] },
+        // turn-level expectedResponse is intentionally present to prove it is dropped (v1).
+        turns: [{ input: { prompt: "hi" }, expectedResponse: { text: "hello" } }],
+      },
+    },
+    { exampleId: "e2", sessionId: "s2" },
+  ],
+  invoked: 2,
+  failed: 0,
+};
+
+const TRACES: SessionTrace[] = [
+  { sessionId: "s1", spans: [], traceIds: ["t1"], toolCallSpanIds: [] },
+  { sessionId: "s2", spans: [], traceIds: ["t2"], toolCallSpanIds: [] },
+];
+
+const EVAL_RESULT: EvaluateResult = { sessionsRequested: 2, sessionsEvaluated: 2, results: [] };
+
+async function run(args: string[], configure?: (core: TestCoreClient) => void) {
+  const core = new TestCoreClient();
+  core.eval.setInvokeDatasetResponse(INVOKE_RESULT);
+  core.eval.setGetTracesResponse(TRACES);
+  core.eval.setEvaluateResponse(EVAL_RESULT);
+  configure?.(core);
+  const io = testIO();
+  const root = createRootHandler(core, {
+    io: io.io,
+    logger: createSilentLogger(),
+    globalConfigAccessor: new TestGlobalConfigAccessor(),
+  });
+  await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]);
+  return { core, stdout: io.stdout() };
+}
+
+const BASE = [
+  "eval",
+  "ondemand",
+  "simulate",
+  "--runtime-id",
+  "r-1",
+  "--payload-template",
+  '{"prompt":"{input}"}',
+  "--dataset",
+  "/tmp/ds.jsonl",
+  "--evaluator",
+  "Builtin.Helpfulness",
+];
+
+describe("eval ondemand simulate", () => {
+  test("registered under ondemand", () => {
+    const io = testIO();
+    const root = createRootHandler(new TestCoreClient(), {
+      io: io.io,
+      logger: createSilentLogger(),
+      globalConfigAccessor: new TestGlobalConfigAccessor(),
+    });
+    // Bare `ondemand` prints help listing both subcommands.
+    expect(root).toBeDefined();
+  });
+
+  test.each([
+    [
+      [
+        "eval",
+        "ondemand",
+        "simulate",
+        "--payload-template",
+        "{}",
+        "--dataset",
+        "d",
+        "--evaluator",
+        "E",
+      ],
+      /--runtime-id/,
+    ],
+    [
+      ["eval", "ondemand", "simulate", "--runtime-id", "r", "--dataset", "d", "--evaluator", "E"],
+      /--payload-template/,
+    ],
+    [
+      [
+        "eval",
+        "ondemand",
+        "simulate",
+        "--runtime-id",
+        "r",
+        "--payload-template",
+        "{}",
+        "--evaluator",
+        "E",
+      ],
+      /--dataset/,
+    ],
+    [
+      [
+        "eval",
+        "ondemand",
+        "simulate",
+        "--runtime-id",
+        "r",
+        "--payload-template",
+        "{}",
+        "--dataset",
+        "d",
+      ],
+      /--evaluator/,
+    ],
+  ])("rejects missing required flag", async (args, expected) => {
+    await expect(run(args)).rejects.toThrow(expected);
+  });
+
+  test("composes invokeDataset → getTracesForAgent → evaluate", async () => {
+    const { core } = await run([...BASE, "--qualifier", "PROD", "--header", "x-a:1"]);
+
+    const invoke = core.eval.calls.find((c) => c.method === "invokeDataset");
+    expect(invoke?.args[0]).toMatchObject({
+      runtimeId: "r-1",
+      qualifier: "PROD",
+      payloadTemplate: '{"prompt":"{input}"}',
+      headers: [["x-a", "1"]],
+      dataset: "/tmp/ds.jsonl",
+    });
+    // Ctrl-C signal threaded to the replay.
+    expect(invoke?.args[2]).toBeInstanceOf(AbortSignal);
+
+    // getTracesForAgent is asked for exactly the sessions the replay created.
+    const traces = core.eval.calls.find((c) => c.method === "getTracesForAgent");
+    expect(traces?.args[0]).toMatchObject({
+      agent: "r-1",
+      endpoint: "PROD",
+      sessionIds: ["s1", "s2"],
+    });
+
+    // evaluate gets the runtime evaluators over the gathered traces.
+    const evaluate = core.eval.calls.find((c) => c.method === "evaluate");
+    expect(evaluate?.args[0]).toMatchObject({ evaluatorIds: ["Builtin.Helpfulness"] });
+  });
+
+  // Golden: the adapted EvaluationReferenceInput[] handed to evaluate. Locks session-level
+  // assertions/trajectory (1:1), the sessionId correlation, and that e2 (no GT) and the
+  // turn-level expectedResponse both contribute nothing.
+  test("adapts inline ground truth to EvaluationReferenceInput [golden]", async () => {
+    const { core } = await run(BASE);
+    const evaluate = core.eval.calls.find((c) => c.method === "evaluate");
+    expect(evaluate).toBeDefined();
+    const input = evaluate!.args[0] as { groundTruth: unknown };
+    matchGolden(
+      FIXTURES,
+      "ondemand-simulate-groundtruth.golden.json",
+      JSON.stringify(input.groundTruth, null, 2),
+    );
+  });
+
+  test("renders scores inline with invoked/failed counts [golden]", async () => {
+    const { stdout } = await run(BASE);
+    matchGolden(FIXTURES, "ondemand-simulate-output.golden.json", stdout);
+  });
+
+  test("refuses to evaluate when nothing was invoked", async () => {
+    await expect(
+      run(BASE, (core) =>
+        core.eval.setInvokeDatasetResponse({ sessions: [], invoked: 0, failed: 3 }),
+      ),
+    ).rejects.toThrow(/no examples could be invoked \(3 failed\)/);
+  });
+});