feat(eval): imperative online-insight commands + BYO execution role

src/core/eval.tsx

@@ -101,6 +101,7 @@ import type {
   CreateConfigurationBundleInput,
   CreateDatasetInput,
   CreateOnlineEvalInput,
+  CreateOnlineInsightInput,
   EvaluateInput,
   EvaluateResult,
   GetBatchEvaluationResult,
@@ -556,6 +557,57 @@ export class EvalClient implements CoreEvalClient {
       : retryWhileRolePropagates(() => control.send(command));
   }
 
+  async createOnlineInsight(
+    input: CreateOnlineInsightInput,
+    options: CoreOptions,
+  ): Promise<CreateOnlineEvaluationConfigResponse> {
+    const dataSourceConfig =
+      input.agent !== undefined
+        ? await agentDataSource(input.agent, input.endpoint, this.clients, options)
+        : input.dataSourceConfig;
+    const control = this.clients.control(toClientConfig(options));
+
+    const command = new CreateOnlineEvaluationConfigCommand({
+      onlineEvaluationConfigName: input.name,
+      description: input.description,
+      rule: toRule(input.samplingRate, input.sessionTimeoutMinutes, input.filters),
+      dataSourceConfig,
+      insights: input.insightIds.map((insightId) => ({ insightId })),
+      clusteringConfig: input.clusteringConfig,
+      evaluationExecutionRoleArn: input.evaluationExecutionRoleArn,
+      enableOnCreate: input.enableOnCreate ?? true,
+    });
+
+    return control.send(command);
+  }
+
+  getOnlineInsight(id: string, options: CoreOptions): Promise<GetOnlineEvaluationConfigResponse> {
+    return this.getOnlineEvaluationConfig(id, options);
+  }
+
+  listOnlineInsights(
+    nextToken: string | undefined,
+    maxResults: number | undefined,
+    options: CoreOptions,
+  ): Promise<ListOnlineEvaluationConfigsResponse> {
+    return this.listOnlineEvaluationConfigs(nextToken, maxResults, options);
+  }
+
+  setOnlineInsightExecutionStatus(
+    id: string,
+    executionStatus: "ENABLED" | "DISABLED",
+    options: CoreOptions,
+  ): Promise<UpdateOnlineEvaluationConfigResponse> {
+    return this.setOnlineEvaluationExecutionStatus(id, executionStatus, options);
+  }
+
+  deleteOnlineInsight(
+    id: string,
+    options: CoreOptions,
+  ): Promise<DeleteOnlineEvaluationConfigResponse> {
+    return this.deleteOnlineEvaluationConfig(id, options);
+  }
+
   // updateOnlineEvaluationConfig fetches the current config and merges the
   // provided fields over it, because UpdateOnlineEvaluationConfig replaces the
   // whole `rule` (and, when endpoint changes, `dataSourceConfig`) rather than

src/handlers/eval/index.tsx

@@ -5,6 +5,7 @@ import type { AppIO } from "../../io";
 import type { Core } from "../types";
 import { createEvaluatorHandler } from "./evaluator";
 import { createOnlineEvalHandler } from "./online-eval";
+import { createOnlineInsightHandler } from "./online-insight";
 import { createDatasetHandler } from "./dataset";
 import { createBatchEvaluationHandler } from "./batch-evaluation";
 import { createOnDemandHandler } from "./ondemand";
@@ -16,6 +17,7 @@ export function createEvalHandler(core: Core, io: AppIO): Router {
     .default(renderTui(core, io))
     .handler(createEvaluatorHandler(core, io))
     .handler(createOnlineEvalHandler(core, io))
+    .handler(createOnlineInsightHandler(core, io))
     .handler(createDatasetHandler(core, io))
     .handler(createBatchEvaluationHandler(core, io))
     .handler(createOnDemandHandler(core, io))

src/handlers/eval/online-insight/__fixtures__/CreateOnlineEvaluationConfigCommand.571c17181e8993e5.json

@@ -0,0 +1,14 @@
+{
+  "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-ybMHPI7ui3",
+  "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-ybMHPI7ui3",
+  "createdAt": {
+    "$date": "2026-08-22T15:09:11.820Z"
+  },
+  "status": "CREATING",
+  "executionStatus": "DISABLED",
+  "outputConfig": {
+    "cloudWatchConfig": {
+      "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_insight_fixture-ybMHPI7ui3"
+    }
+  }
+}

src/handlers/eval/online-insight/__fixtures__/DeleteOnlineEvaluationConfigCommand.85b6716a8b666253.json

@@ -0,0 +1,5 @@
+{
+  "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-ybMHPI7ui3",
+  "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-ybMHPI7ui3",
+  "status": "DELETING"
+}

src/handlers/eval/online-insight/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json

@@ -0,0 +1,46 @@
+{
+  "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:runtime/testAgent_Agent-wm9hYBD93Y",
+  "agentRuntimeName": "testAgent_Agent",
+  "agentRuntimeId": "testAgent_Agent-wm9hYBD93Y",
+  "agentRuntimeVersion": "3",
+  "createdAt": {
+    "$date": "2026-03-24T16:02:00.301Z"
+  },
+  "lastUpdatedAt": {
+    "$date": "2026-03-24T16:13:08.198Z"
+  },
+  "roleArn": "arn:aws:iam::725476964917:role/AgentCore-myimport-defaul-ApplicationAgentTestAgent-tJFjyd6jLIOn",
+  "networkConfiguration": {
+    "networkMode": "PUBLIC"
+  },
+  "status": "READY",
+  "lifecycleConfiguration": {
+    "idleRuntimeSessionTimeout": 900,
+    "maxLifetime": 28800
+  },
+  "description": "AgentCore Runtime: myimport_testAgent_Agent",
+  "workloadIdentityDetails": {
+    "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:workload-identity-directory/default/workload-identity/testAgent_Agent-wm9hYBD93Y"
+  },
+  "agentRuntimeArtifact": {
+    "codeConfiguration": {
+      "code": {
+        "s3": {
+          "bucket": "cdk-hnb659fds-assets-725476964917-us-west-2",
+          "prefix": "5f58024b207272eeb8560702468b402cb3aeea1c8ee36d3c5f348fcb471bb58f.zip"
+        }
+      },
+      "runtime": "PYTHON_3_10",
+      "entryPoint": [
+        "opentelemetry-instrument",
+        "main.py"
+      ]
+    }
+  },
+  "environmentVariables": {
+    "MEMORY_TESTAGENT_AGENT_MEM_ID": "testAgent_Agent_mem-17a3Lg8yrL"
+  },
+  "metadataConfiguration": {
+    "requireMMDSV2": true
+  }
+}

src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.859097834b7e85cf.json

@@ -0,0 +1,6 @@
+{
+  "$error": {
+    "name": "ResourceNotFoundException",
+    "message": "Online evaluation configuration not found"
+  }
+}

src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.85b6716a8b666253.json

@@ -0,0 +1,40 @@
+{
+  "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-ybMHPI7ui3",
+  "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-ybMHPI7ui3",
+  "onlineEvaluationConfigName": "agentcore_cli_online_insight_fixture",
+  "rule": {
+    "samplingConfig": {
+      "samplingPercentage": 10
+    }
+  },
+  "dataSourceConfig": {
+    "cloudWatchLogs": {
+      "logGroupNames": [
+        "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT"
+      ],
+      "serviceNames": [
+        "testAgent_Agent.DEFAULT"
+      ]
+    }
+  },
+  "status": "ACTIVE",
+  "executionStatus": "DISABLED",
+  "createdAt": {
+    "$date": "2026-08-22T15:09:11.820Z"
+  },
+  "updatedAt": {
+    "$date": "2026-08-22T15:09:12.022Z"
+  },
+  "evaluators": [],
+  "insights": [
+    {
+      "insightId": "Builtin.Insight.FailureAnalysis"
+    }
+  ],
+  "outputConfig": {
+    "cloudWatchConfig": {
+      "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_insight_fixture-ybMHPI7ui3"
+    }
+  },
+  "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreEvalsSDK-us-west-2-a6864eb339"
+}