From 2552ac0b5fd6da50a18cdd5bbf7980375a68230d Mon Sep 17 00:00:00 2001 From: jariy17 Date: Sat, 22 Aug 2026 15:52:31 +0000 Subject: [PATCH] feat(eval): online-insight TUI get/list + insights column (P4, #2029) --- src/components/OnlineEvalPicker.tsx | 48 ++-- src/components/Root.tsx | 26 +++ .../online-eval/online-eval.screen.test.tsx | 20 ++ .../eval/online-insight/get/screen.tsx | 77 +++++++ src/handlers/eval/online-insight/index.tsx | 7 + .../eval/online-insight/list/screen.tsx | 28 +++ .../online-insight.screen.test.tsx | 206 ++++++++++++++++++ src/handlers/eval/online-insight/screen.tsx | 6 + 8 files changed, 404 insertions(+), 14 deletions(-) create mode 100644 src/handlers/eval/online-insight/get/screen.tsx create mode 100644 src/handlers/eval/online-insight/list/screen.tsx create mode 100644 src/handlers/eval/online-insight/online-insight.screen.test.tsx create mode 100644 src/handlers/eval/online-insight/screen.tsx diff --git a/src/components/OnlineEvalPicker.tsx b/src/components/OnlineEvalPicker.tsx index df8141c15..9bbb6de3b 100644 --- a/src/components/OnlineEvalPicker.tsx +++ b/src/components/OnlineEvalPicker.tsx @@ -3,19 +3,20 @@ import { useNavigate } from "react-router"; import type { ScreenProps } from "../handlers/types"; import { coreOptsFromCtx } from "../handlers/utils"; import { formatTimestamp } from "./formatTimestamp"; -import { PaginatedTablePicker } from "./PaginatedTablePicker"; +import { PaginatedTablePicker, type TokenPage } from "./PaginatedTablePicker"; import type { DataTableColumn } from "./ui/data-table"; // OnlineEvalRow is the flat, display-ready shape the table renders. It also // satisfies DataTable's `T extends Record` constraint, which the // SDK's OnlineEvaluationConfigSummary interface does not. The list API returns -// only summary fields (name/status/executionStatus/timestamps); richer detail -// like sampling rate and evaluators comes from GetOnlineEvaluationConfig. +// only summary fields (name/status/executionStatus/timestamps/insights); richer +// detail like sampling rate and evaluators comes from GetOnlineEvaluationConfig. interface OnlineEvalRow extends Record { configId: string; configName: string; status: string; executionStatus: string; + hasInsights: boolean; updatedAt: string; } @@ -23,6 +24,9 @@ export const onlineEvalColumns = [ { key: "configName", header: "name", flex: true }, { key: "status", header: "status", width: 12 }, { key: "executionStatus", header: "execution", width: 11 }, + // #2029: surface whether the config has insights enabled straight from the + // list summary — no per-row GetOnlineEvaluationConfig call needed. + { key: "hasInsights", header: "insights", width: 9, render: (value) => (value ? "yes" : "-") }, { key: "updatedAt", header: "updated UTC", @@ -38,6 +42,7 @@ function toRow(config: OnlineEvaluationConfigSummary): OnlineEvalRow { configName: config.onlineEvaluationConfigName ?? id, status: config.status ?? "-", executionStatus: config.executionStatus ?? "-", + hasInsights: (config.insights?.length ?? 0) > 0, updatedAt: config.updatedAt?.toISOString() ?? "-", }; } @@ -47,6 +52,15 @@ export interface OnlineEvalPickerProps extends ScreenProps { description?: string; onSelect: (configId: string) => void; onEscape?: () => void; + // The online-insight list reuses this picker (same columns, incl. the #2029 + // insights column) but reads from a different Core method and namespace, so + // the data source and the noun in the status messages are overridable. + resourceLabel?: string; + queryKey?: readonly unknown[]; + loadPage?: ( + token: string | undefined, + pageSize: number, + ) => Promise>; } /** @@ -62,6 +76,9 @@ export function OnlineEvalPicker({ description, onSelect, onEscape, + resourceLabel = "online evaluation configs", + queryKey, + loadPage, }: OnlineEvalPickerProps) { const opts = coreOptsFromCtx(ctx); const navigate = useNavigate(); @@ -71,23 +88,26 @@ export function OnlineEvalPicker({ { - const response = await core.eval.listOnlineEvaluationConfigs(token, pageSize, opts); - return { - items: response.onlineEvaluationConfigs ?? [], - nextToken: response.nextToken, - }; - }} + queryKey={queryKey ?? ["online-evals", opts.region]} + loadPage={ + loadPage ?? + (async (token, pageSize) => { + const response = await core.eval.listOnlineEvaluationConfigs(token, pageSize, opts); + return { + items: response.onlineEvaluationConfigs ?? [], + nextToken: response.nextToken, + }; + }) + } toRow={toRow} columns={onlineEvalColumns} getValue={(row) => row.configId} onSelect={onSelect} onBack={goBack} - loadingMessage="Loading online evaluation configs…" + loadingMessage={`Loading ${resourceLabel}…`} errorMessage={(error) => `Error: ${error.message}`} - emptyMessage="No online evaluation configs found in this Region." - emptyPageMessage="No online evaluation configs on this page." + emptyMessage={`No ${resourceLabel} found in this Region.`} + emptyPageMessage={`No ${resourceLabel} on this page.`} /> ); } diff --git a/src/components/Root.tsx b/src/components/Root.tsx index 560096e06..3fee1e850 100644 --- a/src/components/Root.tsx +++ b/src/components/Root.tsx @@ -48,6 +48,12 @@ import { OnlineEvalGetScreen, OnlineEvalGetJsonScreen, } from "../handlers/eval/online-eval/get/screen.tsx"; +import { OnlineInsightScreen } from "../handlers/eval/online-insight/screen.tsx"; +import { OnlineInsightListScreen } from "../handlers/eval/online-insight/list/screen.tsx"; +import { + OnlineInsightGetScreen, + OnlineInsightGetJsonScreen, +} from "../handlers/eval/online-insight/get/screen.tsx"; import { BatchEvaluationScreen } from "../handlers/eval/batch-evaluation/screen.tsx"; import { BatchEvaluationListScreen } from "../handlers/eval/batch-evaluation/list/screen.tsx"; import { BatchEvaluationGetJsonScreen } from "../handlers/eval/batch-evaluation/get/screen.tsx"; @@ -482,6 +488,26 @@ export function Root({ path, ctx, core, queryClient }: RootProps) { path="agentcore/eval/online-eval/get/:configId/json" element={} /> + } + /> + } + /> + } + /> + } + /> + } + /> } /> { expect(frame).toContain("2026-07-21 02:03"); }); + test("shows the #2029 insights column: 'yes' when enabled, '-' when not", async () => { + const core = coreWithConfigs([ + configSummary({ + onlineEvaluationConfigName: "with_insights", + insights: [{ insightId: "ins-1" }], + }), + ]); + const screen = renderScreen("/agentcore/eval/online-eval/list", { core }); + + await waitForText(screen.lastFrame, "with_insights"); + expect(screen.lastFrame()).toContain("insights"); + expect(screen.lastFrame()).toContain("yes"); + + const noInsights = renderScreen("/agentcore/eval/online-eval/list", { + core: coreWithConfigs([configSummary({ onlineEvaluationConfigName: "no_insights" })]), + }); + await waitForText(noInsights.lastFrame, "no_insights"); + expect(noInsights.lastFrame()).not.toContain("yes"); + }); + test("calls listOnlineEvaluationConfigs with exact Core options", async () => { const core = coreWithConfigs([configSummary()]); renderScreen("/agentcore/eval/online-eval/list", { core, endpointUrl: evalEndpointUrl }); diff --git a/src/handlers/eval/online-insight/get/screen.tsx b/src/handlers/eval/online-insight/get/screen.tsx new file mode 100644 index 000000000..259b8df85 --- /dev/null +++ b/src/handlers/eval/online-insight/get/screen.tsx @@ -0,0 +1,77 @@ +import { useQuery } from "@tanstack/react-query"; +import { useNavigate, useParams } from "react-router"; +import { JsonDetail } from "../../../../components/JsonDetail"; +import { ResourceDetailScreen } from "../../../../components/ResourceDetailScreen"; +import type { ScreenProps } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +function useOnlineInsightDetail({ ctx, core }: ScreenProps, configId: string | undefined) { + const opts = coreOptsFromCtx(ctx); + return useQuery({ + queryKey: ["online-insight", opts.region, configId], + queryFn: () => core.eval.getOnlineInsight(configId!, opts), + enabled: configId !== undefined, + }); +} + +export function OnlineInsightGetScreen(props: ScreenProps) { + const navigate = useNavigate(); + const { configId } = useParams(); + const detail = useOnlineInsightDetail(props, configId); + const config = detail.data; + const samplingPercentage = config?.rule?.samplingConfig?.samplingPercentage; + const insightCount = config?.insights?.length ?? 0; + const frequencies = config?.clusteringConfig?.frequencies ?? []; + + return ( + 0 ? insightCount.toString() : "-", + clustering: frequencies.length > 0 ? frequencies.join(", ") : "-", + ...(config?.failureReason ? { failureReason: config.failureReason } : {}), + role: config?.evaluationExecutionRoleArn ?? "-", + }} + actions={ + configId && config + ? [ + { + name: "detail", + description: "show the full JSON (insights, clustering, filters, data source)", + onSelect: () => + navigate( + `/agentcore/eval/online-insight/get/${encodeURIComponent(configId)}/json`, + ), + }, + ] + : [] + } + loadingLabel="Loading online insight config…" + onRetry={() => void detail.refetch()} + selectLabel="open detail" + /> + ); +} + +export function OnlineInsightGetJsonScreen(props: ScreenProps) { + const { configId } = useParams(); + const detail = useOnlineInsightDetail(props, configId); + + return ( + void detail.refetch()} + /> + ); +} diff --git a/src/handlers/eval/online-insight/index.tsx b/src/handlers/eval/online-insight/index.tsx index b9a3c5108..b54329b49 100644 --- a/src/handlers/eval/online-insight/index.tsx +++ b/src/handlers/eval/online-insight/index.tsx @@ -1,4 +1,6 @@ import { Router } from "../../../router"; +import { renderTui } from "../../../tui"; +import { withTuiOnEmptyFlagsAndArgs } from "../../../middleware"; import type { AppIO } from "../../../io"; import type { Core } from "../../types"; import { createCreateOnlineInsightHandler } from "./create"; @@ -11,6 +13,9 @@ import { createDeleteOnlineInsightHandler } from "./delete"; export function createOnlineInsightHandler(core: Core, io: AppIO): Router { return new Router("online-insight", "manage AgentCore online insight configs") + .use(withTuiOnEmptyFlagsAndArgs(core, io)) + .default(renderTui(core, io)) + .supportedTuiCommands("get", "list") .handler(createCreateOnlineInsightHandler(core, io)) .handler(createGetOnlineInsightHandler(core)) .handler(createListOnlineInsightHandler(core)) @@ -19,3 +24,5 @@ export function createOnlineInsightHandler(core: Core, io: AppIO): Router { .handler(createResumeOnlineInsightHandler(core)) .handler(createDeleteOnlineInsightHandler(core)); } + +export { OnlineInsightScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/online-insight/list/screen.tsx b/src/handlers/eval/online-insight/list/screen.tsx new file mode 100644 index 000000000..24bf417be --- /dev/null +++ b/src/handlers/eval/online-insight/list/screen.tsx @@ -0,0 +1,28 @@ +import { useNavigate } from "react-router"; +import { OnlineEvalPicker } from "../../../../components/OnlineEvalPicker"; +import type { ScreenProps } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export function OnlineInsightListScreen(props: ScreenProps) { + const navigate = useNavigate(); + const opts = coreOptsFromCtx(props.ctx); + + return ( + { + const response = await props.core.eval.listOnlineInsights(token, pageSize, opts); + return { + items: response.onlineEvaluationConfigs ?? [], + nextToken: response.nextToken, + }; + }} + onSelect={(configId) => + navigate(`/agentcore/eval/online-insight/get/${encodeURIComponent(configId)}`) + } + /> + ); +} diff --git a/src/handlers/eval/online-insight/online-insight.screen.test.tsx b/src/handlers/eval/online-insight/online-insight.screen.test.tsx new file mode 100644 index 000000000..e82b46cc4 --- /dev/null +++ b/src/handlers/eval/online-insight/online-insight.screen.test.tsx @@ -0,0 +1,206 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import type { + GetOnlineEvaluationConfigResponse, + OnlineEvaluationConfigSummary, +} from "@aws-sdk/client-bedrock-agentcore-control"; +import { + cleanupScreens, + renderScreen, + TestCoreClient, + waitFor, + waitForText, +} from "../../../testing"; + +afterEach(cleanupScreens); + +const evalEndpointUrl = "https://eval.test"; + +function configSummary( + overrides: Partial = {}, +): OnlineEvaluationConfigSummary { + return { + onlineEvaluationConfigArn: + "arn:aws:bedrock-agentcore:us-east-1:123456789012:online-evaluation-config/oic-1", + onlineEvaluationConfigId: "oic-1", + onlineEvaluationConfigName: "prod_failure_insights", + status: "ACTIVE", + executionStatus: "ENABLED", + insights: [{ insightId: "ins-failure" }], + createdAt: new Date("2026-07-19T01:02:03.000Z"), + updatedAt: new Date("2026-07-20T12:34:56.000Z"), + ...overrides, + }; +} + +function getConfigResponse( + overrides: Partial = {}, +): GetOnlineEvaluationConfigResponse { + return { + onlineEvaluationConfigArn: + "arn:aws:bedrock-agentcore:us-east-1:123456789012:online-evaluation-config/oic-1", + onlineEvaluationConfigId: "oic-1", + onlineEvaluationConfigName: "prod_failure_insights", + status: "ACTIVE", + executionStatus: "ENABLED", + rule: { samplingConfig: { samplingPercentage: 5 } }, + dataSourceConfig: { + cloudWatchLogs: { logGroupNames: ["/aws/bedrock-agentcore/runtime/x"], serviceNames: [] }, + }, + insights: [{ insightId: "ins-failure" }, { insightId: "ins-intent" }], + clusteringConfig: { frequencies: ["DAILY", "WEEKLY"] }, + evaluationExecutionRoleArn: "arn:aws:iam::123456789012:role/online-insight-role", + createdAt: new Date("2026-07-19T01:02:03.000Z"), + updatedAt: new Date("2026-07-20T12:34:56.000Z"), + ...overrides, + }; +} + +function coreWithConfigs(configs: OnlineEvaluationConfigSummary[]): TestCoreClient { + const core = new TestCoreClient(); + core.eval.setOnlineEvalListResponse({ onlineEvaluationConfigs: configs }); + return core; +} + +describe("online-insight menu", () => { + test("offers only the read-only commands", async () => { + const screen = renderScreen("/agentcore/eval/online-insight"); + + await waitForText(screen.lastFrame, "get an online insight config by id"); + const frame = screen.lastFrame()!; + expect(frame).toContain("list"); + expect(frame).not.toContain("create"); + expect(frame).not.toContain("update"); + expect(frame).not.toContain("pause"); + expect(frame).not.toContain("resume"); + expect(frame).not.toContain("delete"); + }); +}); + +describe("online-insight picker", () => { + test("renders name, execution status, the insights column, and update time", async () => { + const core = coreWithConfigs([ + configSummary({ + onlineEvaluationConfigName: "staging_intent_insights", + executionStatus: "DISABLED", + updatedAt: new Date("2026-07-21T02:03:04.000Z"), + }), + ]); + const screen = renderScreen("/agentcore/eval/online-insight/list", { core }); + + await waitForText(screen.lastFrame, "staging_intent_insights"); + const frame = screen.lastFrame()!; + expect(frame).toContain("insights"); + expect(frame).toContain("DISABLED"); + expect(frame).toContain("yes"); + expect(frame).toContain("2026-07-21 02:03"); + }); + + test("shows '-' in the insights column when a config has no insights", async () => { + const core = coreWithConfigs([ + configSummary({ onlineEvaluationConfigName: "no_insights_config", insights: [] }), + ]); + const screen = renderScreen("/agentcore/eval/online-insight/list", { core }); + + await waitForText(screen.lastFrame, "no_insights_config"); + expect(screen.lastFrame()).not.toContain("yes"); + }); + + test("calls listOnlineInsights with exact Core options", async () => { + const core = coreWithConfigs([configSummary()]); + renderScreen("/agentcore/eval/online-insight/list", { core, endpointUrl: evalEndpointUrl }); + + await waitFor(() => core.eval.calls.some((call) => call.method === "listOnlineInsights")); + expect(core.eval.calls.filter((call) => call.method === "listOnlineInsights")).toEqual([ + { + method: "listOnlineInsights", + args: [ + undefined, + expect.any(Number), + { region: "us-east-1", endpointUrl: evalEndpointUrl }, + ], + }, + ]); + }); + + test("bare online-insight get redirects to the picker", async () => { + const core = coreWithConfigs([ + configSummary({ + onlineEvaluationConfigId: "redirected-oic", + onlineEvaluationConfigName: "redirected_insight", + }), + ]); + const screen = renderScreen("/agentcore/eval/online-insight/get", { core }); + + await waitForText(screen.lastFrame, "redirected_insight"); + expect(core.eval.calls[0]?.method).toBe("listOnlineInsights"); + }); + + test("selection opens the matching config detail", async () => { + const core = coreWithConfigs([configSummary({ onlineEvaluationConfigId: "oic-1" })]); + core.eval.setOnlineEvalGetResponse(getConfigResponse({ onlineEvaluationConfigId: "oic-1" })); + const screen = renderScreen("/agentcore/eval/online-insight/list", { core }); + + await waitForText(screen.lastFrame, "prod_failure_insights"); + await screen.press("return"); + await waitForText(screen.lastFrame, "agentcore → eval → online-insight → get → oic-1"); + await waitFor(() => + core.eval.calls.some( + (call) => call.method === "getOnlineInsight" && call.args[0] === "oic-1", + ), + ); + }); + + test("shows the empty state", async () => { + const empty = renderScreen("/agentcore/eval/online-insight/list"); + await waitForText(empty.lastFrame, "No online insight configs found in this Region."); + }); +}); + +describe("online-insight detail", () => { + test("renders sampling, execution status, insight count, and clustering frequencies", async () => { + const core = new TestCoreClient(); + core.eval.setOnlineEvalGetResponse(getConfigResponse()); + const screen = renderScreen("/agentcore/eval/online-insight/get/oic-1", { + core, + endpointUrl: evalEndpointUrl, + }); + + await waitForText(screen.lastFrame, "show the full JSON"); + const frame = screen.lastFrame()!; + expect(frame).toContain("prod_failure_insights"); + expect(frame).toMatch(/sampling\s+5%/); + expect(frame).toMatch(/execution\s+ENABLED/); + expect(frame).toMatch(/insights\s+2/); + expect(frame).toContain("DAILY, WEEKLY"); + expect(frame).not.toContain("evaluators"); + expect(core.eval.calls.find((call) => call.method === "getOnlineInsight")).toEqual({ + method: "getOnlineInsight", + args: ["oic-1", { region: "us-east-1", endpointUrl: evalEndpointUrl }], + }); + }); + + test("opens the complete config JSON", async () => { + const core = new TestCoreClient(); + core.eval.setOnlineEvalGetResponse(getConfigResponse()); + const screen = renderScreen("/agentcore/eval/online-insight/get/oic-1", { core }); + + await waitForText(screen.lastFrame, "show the full JSON"); + await screen.press("return"); + await waitForText(screen.lastFrame, "agentcore → eval → online-insight → get → oic-1 → json"); + expect(screen.lastFrame()).toContain('"clusteringConfig"'); + }); + + test("retries a failed detail query", async () => { + const core = new TestCoreClient(); + core.eval.setError(new Error("insight unavailable")); + const screen = renderScreen("/agentcore/eval/online-insight/get/oic-1", { core }); + + await waitForText(screen.lastFrame, "insight unavailable"); + expect(screen.lastFrame()).toContain("[r] retry"); + + core.eval.setError(undefined); + core.eval.setOnlineEvalGetResponse(getConfigResponse()); + await screen.write("r"); + await waitForText(screen.lastFrame, "show the full JSON"); + }); +}); diff --git a/src/handlers/eval/online-insight/screen.tsx b/src/handlers/eval/online-insight/screen.tsx new file mode 100644 index 000000000..68277b04e --- /dev/null +++ b/src/handlers/eval/online-insight/screen.tsx @@ -0,0 +1,6 @@ +import { RouterScreen } from "../../../components/RouterScreen"; +import type { ScreenProps } from "../../types"; + +export function OnlineInsightScreen(props: ScreenProps) { + return ; +}