diff --git a/src/core/eval.tsx b/src/core/eval.tsx index aef88d11d..3171aacc3 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -101,6 +101,7 @@ import type { CreateConfigurationBundleInput, CreateDatasetInput, CreateOnlineEvalInput, + CreateOnlineInsightInput, EvaluateInput, EvaluateResult, GetBatchEvaluationResult, @@ -556,6 +557,57 @@ export class EvalClient implements CoreEvalClient { : retryWhileRolePropagates(() => control.send(command)); } + async createOnlineInsight( + input: CreateOnlineInsightInput, + options: CoreOptions, + ): Promise { + const dataSourceConfig = + input.agent !== undefined + ? await agentDataSource(input.agent, input.endpoint, this.clients, options) + : input.dataSourceConfig; + const control = this.clients.control(toClientConfig(options)); + + const command = new CreateOnlineEvaluationConfigCommand({ + onlineEvaluationConfigName: input.name, + description: input.description, + rule: toRule(input.samplingRate, input.sessionTimeoutMinutes, input.filters), + dataSourceConfig, + insights: input.insightIds.map((insightId) => ({ insightId })), + clusteringConfig: input.clusteringConfig, + evaluationExecutionRoleArn: input.evaluationExecutionRoleArn, + enableOnCreate: input.enableOnCreate ?? true, + }); + + return control.send(command); + } + + getOnlineInsight(id: string, options: CoreOptions): Promise { + return this.getOnlineEvaluationConfig(id, options); + } + + listOnlineInsights( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + return this.listOnlineEvaluationConfigs(nextToken, maxResults, options); + } + + setOnlineInsightExecutionStatus( + id: string, + executionStatus: "ENABLED" | "DISABLED", + options: CoreOptions, + ): Promise { + return this.setOnlineEvaluationExecutionStatus(id, executionStatus, options); + } + + deleteOnlineInsight( + id: string, + options: CoreOptions, + ): Promise { + return this.deleteOnlineEvaluationConfig(id, options); + } + // updateOnlineEvaluationConfig fetches the current config and merges the // provided fields over it, because UpdateOnlineEvaluationConfig replaces the // whole `rule` (and, when endpoint changes, `dataSourceConfig`) rather than diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx index add16936e..2a9737802 100644 --- a/src/handlers/eval/index.tsx +++ b/src/handlers/eval/index.tsx @@ -5,6 +5,7 @@ import type { AppIO } from "../../io"; import type { Core } from "../types"; import { createEvaluatorHandler } from "./evaluator"; import { createOnlineEvalHandler } from "./online-eval"; +import { createOnlineInsightHandler } from "./online-insight"; import { createDatasetHandler } from "./dataset"; import { createBatchEvaluationHandler } from "./batch-evaluation"; import { createOnDemandHandler } from "./ondemand"; @@ -16,6 +17,7 @@ export function createEvalHandler(core: Core, io: AppIO): Router { .default(renderTui(core, io)) .handler(createEvaluatorHandler(core, io)) .handler(createOnlineEvalHandler(core, io)) + .handler(createOnlineInsightHandler(core, io)) .handler(createDatasetHandler(core, io)) .handler(createBatchEvaluationHandler(core, io)) .handler(createOnDemandHandler(core, io)) diff --git a/src/handlers/eval/online-insight/__fixtures__/CreateOnlineEvaluationConfigCommand.571c17181e8993e5.json b/src/handlers/eval/online-insight/__fixtures__/CreateOnlineEvaluationConfigCommand.571c17181e8993e5.json new file mode 100644 index 000000000..c89867964 --- /dev/null +++ b/src/handlers/eval/online-insight/__fixtures__/CreateOnlineEvaluationConfigCommand.571c17181e8993e5.json @@ -0,0 +1,14 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "createdAt": { + "$date": "2026-08-22T15:09:11.820Z" + }, + "status": "CREATING", + "executionStatus": "DISABLED", + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_insight_fixture-ybMHPI7ui3" + } + } +} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/DeleteOnlineEvaluationConfigCommand.85b6716a8b666253.json b/src/handlers/eval/online-insight/__fixtures__/DeleteOnlineEvaluationConfigCommand.85b6716a8b666253.json new file mode 100644 index 000000000..c95c493eb --- /dev/null +++ b/src/handlers/eval/online-insight/__fixtures__/DeleteOnlineEvaluationConfigCommand.85b6716a8b666253.json @@ -0,0 +1,5 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json b/src/handlers/eval/online-insight/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json new file mode 100644 index 000000000..5e8b39035 --- /dev/null +++ b/src/handlers/eval/online-insight/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json @@ -0,0 +1,46 @@ +{ + "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:runtime/testAgent_Agent-wm9hYBD93Y", + "agentRuntimeName": "testAgent_Agent", + "agentRuntimeId": "testAgent_Agent-wm9hYBD93Y", + "agentRuntimeVersion": "3", + "createdAt": { + "$date": "2026-03-24T16:02:00.301Z" + }, + "lastUpdatedAt": { + "$date": "2026-03-24T16:13:08.198Z" + }, + "roleArn": "arn:aws:iam::725476964917:role/AgentCore-myimport-defaul-ApplicationAgentTestAgent-tJFjyd6jLIOn", + "networkConfiguration": { + "networkMode": "PUBLIC" + }, + "status": "READY", + "lifecycleConfiguration": { + "idleRuntimeSessionTimeout": 900, + "maxLifetime": 28800 + }, + "description": "AgentCore Runtime: myimport_testAgent_Agent", + "workloadIdentityDetails": { + "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:workload-identity-directory/default/workload-identity/testAgent_Agent-wm9hYBD93Y" + }, + "agentRuntimeArtifact": { + "codeConfiguration": { + "code": { + "s3": { + "bucket": "cdk-hnb659fds-assets-725476964917-us-west-2", + "prefix": "5f58024b207272eeb8560702468b402cb3aeea1c8ee36d3c5f348fcb471bb58f.zip" + } + }, + "runtime": "PYTHON_3_10", + "entryPoint": [ + "opentelemetry-instrument", + "main.py" + ] + } + }, + "environmentVariables": { + "MEMORY_TESTAGENT_AGENT_MEM_ID": "testAgent_Agent_mem-17a3Lg8yrL" + }, + "metadataConfiguration": { + "requireMMDSV2": true + } +} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.859097834b7e85cf.json b/src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.859097834b7e85cf.json new file mode 100644 index 000000000..845409c3b --- /dev/null +++ b/src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.859097834b7e85cf.json @@ -0,0 +1,6 @@ +{ + "$error": { + "name": "ResourceNotFoundException", + "message": "Online evaluation configuration not found" + } +} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.85b6716a8b666253.json b/src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.85b6716a8b666253.json new file mode 100644 index 000000000..1eace4cf7 --- /dev/null +++ b/src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.85b6716a8b666253.json @@ -0,0 +1,40 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "onlineEvaluationConfigName": "agentcore_cli_online_insight_fixture", + "rule": { + "samplingConfig": { + "samplingPercentage": 10 + } + }, + "dataSourceConfig": { + "cloudWatchLogs": { + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT" + ], + "serviceNames": [ + "testAgent_Agent.DEFAULT" + ] + } + }, + "status": "ACTIVE", + "executionStatus": "DISABLED", + "createdAt": { + "$date": "2026-08-22T15:09:11.820Z" + }, + "updatedAt": { + "$date": "2026-08-22T15:09:12.022Z" + }, + "evaluators": [], + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_insight_fixture-ybMHPI7ui3" + } + }, + "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreEvalsSDK-us-west-2-a6864eb339" +} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json b/src/handlers/eval/online-insight/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json new file mode 100644 index 000000000..51caef74b --- /dev/null +++ b/src/handlers/eval/online-insight/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json @@ -0,0 +1,465 @@ +{ + "onlineEvaluationConfigs": [ + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigId": "ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigName": "ABVfyLatest_ProdEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-17T22:09:59.115Z" + }, + "updatedAt": { + "$date": "2026-06-17T22:10:11.792Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigId": "ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigName": "ABVfyLatest_StagingEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-17T22:09:59.058Z" + }, + "updatedAt": { + "$date": "2026-06-17T22:10:12.095Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyPrerelease_ProdEval-MtuOYB2GZP", + "onlineEvaluationConfigId": "ABVfyPrerelease_ProdEval-MtuOYB2GZP", + "onlineEvaluationConfigName": "ABVfyPrerelease_ProdEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-17T20:28:10.738Z" + }, + "updatedAt": { + "$date": "2026-06-17T20:28:26.527Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyPrerelease_StagingEval-fH26hNBHL6", + "onlineEvaluationConfigId": "ABVfyPrerelease_StagingEval-fH26hNBHL6", + "onlineEvaluationConfigName": "ABVfyPrerelease_StagingEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-17T20:28:09.163Z" + }, + "updatedAt": { + "$date": "2026-06-17T20:28:26.799Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/EvoBugBash_abOnlineEval-Q0Iv1k2cdo", + "onlineEvaluationConfigId": "EvoBugBash_abOnlineEval-Q0Iv1k2cdo", + "onlineEvaluationConfigName": "EvoBugBash_abOnlineEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-12T22:27:27.466Z" + }, + "updatedAt": { + "$date": "2026-06-12T22:27:40.636Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/GwFilterVfy_ctrlEval-T97o5LAtRS", + "onlineEvaluationConfigId": "GwFilterVfy_ctrlEval-T97o5LAtRS", + "onlineEvaluationConfigName": "GwFilterVfy_ctrlEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T18:27:10.496Z" + }, + "updatedAt": { + "$date": "2026-06-15T18:27:26.982Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/GwFilterVfy_treatEval-bTv4u14rUQ", + "onlineEvaluationConfigId": "GwFilterVfy_treatEval-bTv4u14rUQ", + "onlineEvaluationConfigName": "GwFilterVfy_treatEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T18:27:10.370Z" + }, + "updatedAt": { + "$date": "2026-06-15T18:27:27.240Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/OhioTripPlanner_ProdEvalStagin-8wGIBCFzOE", + "onlineEvaluationConfigId": "OhioTripPlanner_ProdEvalStagin-8wGIBCFzOE", + "onlineEvaluationConfigName": "OhioTripPlanner_ProdEvalStagin", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-16T00:09:44.834Z" + }, + "updatedAt": { + "$date": "2026-06-16T00:09:56.722Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/OhioTripPlanner_StagingOnly-A3o7OsFV5a", + "onlineEvaluationConfigId": "OhioTripPlanner_StagingOnly-A3o7OsFV5a", + "onlineEvaluationConfigName": "OhioTripPlanner_StagingOnly", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-16T00:22:22.611Z" + }, + "updatedAt": { + "$date": "2026-06-16T00:22:44.675Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeBundle-UekzTY3B12", + "onlineEvaluationConfigId": "PromoteT2_oeBundle-UekzTY3B12", + "onlineEvaluationConfigName": "PromoteT2_oeBundle", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T17:06:17.370Z" + }, + "updatedAt": { + "$date": "2026-06-15T17:06:45.741Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeCtrl-Vwz1F69rtu", + "onlineEvaluationConfigId": "PromoteT2_oeCtrl-Vwz1F69rtu", + "onlineEvaluationConfigName": "PromoteT2_oeCtrl", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T17:06:17.504Z" + }, + "updatedAt": { + "$date": "2026-06-15T17:06:44.609Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeT2c-2CHeFHBYfA", + "onlineEvaluationConfigId": "PromoteT2_oeT2c-2CHeFHBYfA", + "onlineEvaluationConfigName": "PromoteT2_oeT2c", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T17:06:17.603Z" + }, + "updatedAt": { + "$date": "2026-06-15T17:06:45.174Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeT2t-pKLwdZDXyw", + "onlineEvaluationConfigId": "PromoteT2_oeT2t-pKLwdZDXyw", + "onlineEvaluationConfigName": "PromoteT2_oeT2t", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T17:06:17.784Z" + }, + "updatedAt": { + "$date": "2026-06-15T17:06:45.466Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeTreat-bC8FNM8MEN", + "onlineEvaluationConfigId": "PromoteT2_oeTreat-bC8FNM8MEN", + "onlineEvaluationConfigName": "PromoteT2_oeTreat", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T17:06:24.185Z" + }, + "updatedAt": { + "$date": "2026-06-15T17:06:44.903Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/TEMPAGENTTJAB_MyOnlineEval_prod-n6GVQ2Atnw", + "onlineEvaluationConfigId": "TEMPAGENTTJAB_MyOnlineEval_prod-n6GVQ2Atnw", + "onlineEvaluationConfigName": "TEMPAGENTTJAB_MyOnlineEval_prod", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-05-05T19:22:26.291Z" + }, + "updatedAt": { + "$date": "2026-05-05T19:22:57.162Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/TEMPAGENTTJAB_MyOnlineEval_staging-vPcBdKCji0", + "onlineEvaluationConfigId": "TEMPAGENTTJAB_MyOnlineEval_staging-vPcBdKCji0", + "onlineEvaluationConfigName": "TEMPAGENTTJAB_MyOnlineEval_staging", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-05-05T19:22:33.950Z" + }, + "updatedAt": { + "$date": "2026-05-05T19:22:57.441Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abdemo_evalconfig-ntDb6Q5PLh", + "onlineEvaluationConfigId": "abdemo_evalconfig-ntDb6Q5PLh", + "onlineEvaluationConfigName": "abdemo_evalconfig", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-05-27T22:10:42.270Z" + }, + "updatedAt": { + "$date": "2026-05-27T22:10:51.516Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestdemo_eval_config-Kp7uUyBqZB", + "onlineEvaluationConfigId": "abtestdemo_eval_config-Kp7uUyBqZB", + "onlineEvaluationConfigName": "abtestdemo_eval_config", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-05-26T19:13:52.461Z" + }, + "updatedAt": { + "$date": "2026-05-26T19:14:02.417Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestval_cs_control_eval-seHjGZ7TeH", + "onlineEvaluationConfigId": "abtestval_cs_control_eval-seHjGZ7TeH", + "onlineEvaluationConfigName": "abtestval_cs_control_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-24T22:58:24.637Z" + }, + "updatedAt": { + "$date": "2026-06-24T22:58:38.138Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestval_cs_treatment_eval-AdSjkL4CHu", + "onlineEvaluationConfigId": "abtestval_cs_treatment_eval-AdSjkL4CHu", + "onlineEvaluationConfigName": "abtestval_cs_treatment_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-24T22:58:24.648Z" + }, + "updatedAt": { + "$date": "2026-06-24T22:58:38.393Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ac570c_myonlineeval-6IRBzU3dkS", + "onlineEvaluationConfigId": "ac570c_myonlineeval-6IRBzU3dkS", + "onlineEvaluationConfigName": "ac570c_myonlineeval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-07-06T18:21:13.033Z" + }, + "updatedAt": { + "$date": "2026-07-06T18:21:28.312Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "onlineEvaluationConfigName": "agentcore_cli_online_insight_fixture", + "status": "CREATING", + "executionStatus": "DISABLED", + "createdAt": { + "$date": "2026-08-22T15:09:11.820Z" + }, + "updatedAt": { + "$date": "2026-08-22T15:09:11.820Z" + }, + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ] + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_abtesteval-53zYJX8x4X", + "onlineEvaluationConfigId": "bugbashagent_abtesteval-53zYJX8x4X", + "onlineEvaluationConfigName": "bugbashagent_abtesteval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T18:19:55.627Z" + }, + "updatedAt": { + "$date": "2026-06-15T18:20:08.431Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_lambda_ctrl_eval-EY3GWNCDgH", + "onlineEvaluationConfigId": "bugbashagent_lambda_ctrl_eval-EY3GWNCDgH", + "onlineEvaluationConfigName": "bugbashagent_lambda_ctrl_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T19:41:54.132Z" + }, + "updatedAt": { + "$date": "2026-06-15T19:42:10.231Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_lambda_trtm_eval-IlN5S34Bc5", + "onlineEvaluationConfigId": "bugbashagent_lambda_trtm_eval-IlN5S34Bc5", + "onlineEvaluationConfigName": "bugbashagent_lambda_trtm_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T19:41:54.111Z" + }, + "updatedAt": { + "$date": "2026-06-15T19:42:10.335Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineEval1-QErY9BEWT5", + "onlineEvaluationConfigId": "demoEval_onlineEval1-QErY9BEWT5", + "onlineEvaluationConfigName": "demoEval_onlineEval1", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-07-16T17:26:21.325Z" + }, + "updatedAt": { + "$date": "2026-07-16T17:26:36.933Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineEvalBoto3_20260716180853-tbD9Ob7VUX", + "onlineEvaluationConfigId": "demoEval_onlineEvalBoto3_20260716180853-tbD9Ob7VUX", + "onlineEvaluationConfigName": "demoEval_onlineEvalBoto3_20260716180853", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-07-16T18:08:54.221Z" + }, + "updatedAt": { + "$date": "2026-07-16T18:08:54.477Z" + }, + "description": "boto3-created: onlineEvalBoto3" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineInsights1-QE6As2GTTB", + "onlineEvaluationConfigId": "demoEval_onlineInsights1-QE6As2GTTB", + "onlineEvaluationConfigName": "demoEval_onlineInsights1", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-07-16T17:26:21.625Z" + }, + "updatedAt": { + "$date": "2026-07-16T17:26:37.218Z" + }, + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "clusteringConfig": { + "frequencies": [ + "DAILY" + ] + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineInsightsBoto3_20260716180854-aTLOAI48Jr", + "onlineEvaluationConfigId": "demoEval_onlineInsightsBoto3_20260716180854-aTLOAI48Jr", + "onlineEvaluationConfigName": "demoEval_onlineInsightsBoto3_20260716180854", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-07-16T18:08:54.762Z" + }, + "updatedAt": { + "$date": "2026-07-16T18:08:54.954Z" + }, + "description": "boto3-created: onlineInsightsBoto3", + "insights": [ + { + "insightId": "Builtin.Insight.UserIntent" + } + ], + "clusteringConfig": { + "frequencies": [ + "DAILY" + ] + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/deploytest_myeval-JMv0ihHd30", + "onlineEvaluationConfigId": "deploytest_myeval-JMv0ihHd30", + "onlineEvaluationConfigName": "deploytest_myeval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-04-21T22:47:43.896Z" + }, + "updatedAt": { + "$date": "2026-04-21T22:47:52.237Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/oiVerify1_prodInsights-rPYFuF2WsE", + "onlineEvaluationConfigId": "oiVerify1_prodInsights-rPYFuF2WsE", + "onlineEvaluationConfigName": "oiVerify1_prodInsights", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-08-21T16:49:04.244Z" + }, + "updatedAt": { + "$date": "2026-08-21T16:49:04.485Z" + }, + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ] + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/temp-5Dz9Ox6yMC", + "onlineEvaluationConfigId": "temp-5Dz9Ox6yMC", + "onlineEvaluationConfigName": "temp", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2025-12-02T21:09:17.495Z" + }, + "updatedAt": { + "$date": "2025-12-02T21:09:17.831Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/tjHarnessDerivTest-8rZzgXEHiM", + "onlineEvaluationConfigId": "tjHarnessDerivTest-8rZzgXEHiM", + "onlineEvaluationConfigName": "tjHarnessDerivTest", + "status": "ACTIVE", + "executionStatus": "DISABLED", + "createdAt": { + "$date": "2026-07-27T17:56:04.618Z" + }, + "updatedAt": { + "$date": "2026-07-27T17:56:04.805Z" + } + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/ListOnlineEvaluationConfigsCommand.784737470547e001.json b/src/handlers/eval/online-insight/__fixtures__/ListOnlineEvaluationConfigsCommand.784737470547e001.json new file mode 100644 index 000000000..2e52b7702 --- /dev/null +++ b/src/handlers/eval/online-insight/__fixtures__/ListOnlineEvaluationConfigsCommand.784737470547e001.json @@ -0,0 +1,18 @@ +{ + "onlineEvaluationConfigs": [ + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigId": "ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigName": "ABVfyLatest_StagingEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-17T22:09:59.058Z" + }, + "updatedAt": { + "$date": "2026-06-17T22:10:12.095Z" + } + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAGLk59iR2ZuyIwqJvoOOfSKAAABLTCCASkGCSqGSIb3DQEHBqCCARowggEWAgEAMIIBDwYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxXCy3KLqyS52zCnWMCARCAgeEluEX/bJ0iVrlzeNF36Ljphp15ybvPCs7096T8JI9Z9f/watuBwOrVGD9y0d+6c2TejHblL3fal/VAvuswS6Rv8RDmuvrIY/VzAkmOvuX3CNYbWUaOzsYxYrxTNC5ozK7DgRejtsAP5nOAWvy8rlik3cBQoRuL23yD5FFlOoKg6cE70CcsQg9wBOkn9ImR0Luh/7+f7RZpFGm7v6adK209ZjRzJ+saUqjZowqbh9D2i6qG/PJmYzjFmSsQAmaOsttZpajSLo14wo8glgE3AxV8LsGWQfYCAdf9/9pp0PrDm+Q=" +} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/ListOnlineEvaluationConfigsCommand.7d2e22c637f6b633.json b/src/handlers/eval/online-insight/__fixtures__/ListOnlineEvaluationConfigsCommand.7d2e22c637f6b633.json new file mode 100644 index 000000000..e6d791d15 --- /dev/null +++ b/src/handlers/eval/online-insight/__fixtures__/ListOnlineEvaluationConfigsCommand.7d2e22c637f6b633.json @@ -0,0 +1,18 @@ +{ + "onlineEvaluationConfigs": [ + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigId": "ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigName": "ABVfyLatest_ProdEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-17T22:09:59.115Z" + }, + "updatedAt": { + "$date": "2026-06-17T22:10:11.792Z" + } + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAHS5pIaHJusDcJMahiIVyUIAAABKjCCASYGCSqGSIb3DQEHBqCCARcwggETAgEAMIIBDAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAzVFhbXdksKx2UQ8bYCARCAgd5kgZswjRNA8eS0/H5aIL2+AzD60pHCrIClfPbmyeFmLMiWmkRKRsCLbd0thl2zxJ/uyolMa2jdONNzbq0YAyKkT0fnRa8P7USPIQt4f/+bNtG3Axi5y4t3yWC/icBXF80icPKmE9yADJxh7mjztXlTl37ZadrtgBmB5ATUrkacCyszJ8tsSCclWLP+eI4yehksQ2M0p/9nTWKm5hzBk10oRtrBrWp+epAG2zuMgsX6T6hzEVkmaG2UQLojnLk3jFWVhALIvrFaNhFbg/7B+G+NUYUtacuXfd1Ez2oBDK8=" +} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/UpdateOnlineEvaluationConfigCommand.21c73944c2015d66.json b/src/handlers/eval/online-insight/__fixtures__/UpdateOnlineEvaluationConfigCommand.21c73944c2015d66.json new file mode 100644 index 000000000..22d01db09 --- /dev/null +++ b/src/handlers/eval/online-insight/__fixtures__/UpdateOnlineEvaluationConfigCommand.21c73944c2015d66.json @@ -0,0 +1,9 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "updatedAt": { + "$date": "2026-08-22T15:09:17.878Z" + }, + "status": "UPDATING", + "executionStatus": "DISABLED" +} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/UpdateOnlineEvaluationConfigCommand.60b23c41862d8b7c.json b/src/handlers/eval/online-insight/__fixtures__/UpdateOnlineEvaluationConfigCommand.60b23c41862d8b7c.json new file mode 100644 index 000000000..a5ccf7200 --- /dev/null +++ b/src/handlers/eval/online-insight/__fixtures__/UpdateOnlineEvaluationConfigCommand.60b23c41862d8b7c.json @@ -0,0 +1,9 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "updatedAt": { + "$date": "2026-08-22T15:09:12.639Z" + }, + "status": "UPDATING", + "executionStatus": "ENABLED" +} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/create.golden.json b/src/handlers/eval/online-insight/__fixtures__/create.golden.json new file mode 100644 index 000000000..0a247b055 --- /dev/null +++ b/src/handlers/eval/online-insight/__fixtures__/create.golden.json @@ -0,0 +1,12 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "createdAt": "2026-08-22T15:09:11.820Z", + "status": "CREATING", + "executionStatus": "DISABLED", + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_insight_fixture-ybMHPI7ui3" + } + } +} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/delete.golden.json b/src/handlers/eval/online-insight/__fixtures__/delete.golden.json new file mode 100644 index 000000000..c95c493eb --- /dev/null +++ b/src/handlers/eval/online-insight/__fixtures__/delete.golden.json @@ -0,0 +1,5 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/get.golden.json b/src/handlers/eval/online-insight/__fixtures__/get.golden.json new file mode 100644 index 000000000..83e07f5d1 --- /dev/null +++ b/src/handlers/eval/online-insight/__fixtures__/get.golden.json @@ -0,0 +1,36 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "onlineEvaluationConfigName": "agentcore_cli_online_insight_fixture", + "rule": { + "samplingConfig": { + "samplingPercentage": 10 + } + }, + "dataSourceConfig": { + "cloudWatchLogs": { + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT" + ], + "serviceNames": [ + "testAgent_Agent.DEFAULT" + ] + } + }, + "status": "ACTIVE", + "executionStatus": "DISABLED", + "createdAt": "2026-08-22T15:09:11.820Z", + "updatedAt": "2026-08-22T15:09:12.022Z", + "evaluators": [], + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_insight_fixture-ybMHPI7ui3" + } + }, + "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreEvalsSDK-us-west-2-a6864eb339" +} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/list-page-1.golden.json b/src/handlers/eval/online-insight/__fixtures__/list-page-1.golden.json new file mode 100644 index 000000000..4ef90dcfd --- /dev/null +++ b/src/handlers/eval/online-insight/__fixtures__/list-page-1.golden.json @@ -0,0 +1,14 @@ +{ + "onlineEvaluationConfigs": [ + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigId": "ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigName": "ABVfyLatest_ProdEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-17T22:09:59.115Z", + "updatedAt": "2026-06-17T22:10:11.792Z" + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAHS5pIaHJusDcJMahiIVyUIAAABKjCCASYGCSqGSIb3DQEHBqCCARcwggETAgEAMIIBDAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAzVFhbXdksKx2UQ8bYCARCAgd5kgZswjRNA8eS0/H5aIL2+AzD60pHCrIClfPbmyeFmLMiWmkRKRsCLbd0thl2zxJ/uyolMa2jdONNzbq0YAyKkT0fnRa8P7USPIQt4f/+bNtG3Axi5y4t3yWC/icBXF80icPKmE9yADJxh7mjztXlTl37ZadrtgBmB5ATUrkacCyszJ8tsSCclWLP+eI4yehksQ2M0p/9nTWKm5hzBk10oRtrBrWp+epAG2zuMgsX6T6hzEVkmaG2UQLojnLk3jFWVhALIvrFaNhFbg/7B+G+NUYUtacuXfd1Ez2oBDK8=" +} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/list-page-2.golden.json b/src/handlers/eval/online-insight/__fixtures__/list-page-2.golden.json new file mode 100644 index 000000000..add563343 --- /dev/null +++ b/src/handlers/eval/online-insight/__fixtures__/list-page-2.golden.json @@ -0,0 +1,14 @@ +{ + "onlineEvaluationConfigs": [ + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigId": "ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigName": "ABVfyLatest_StagingEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-17T22:09:59.058Z", + "updatedAt": "2026-06-17T22:10:12.095Z" + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAGLk59iR2ZuyIwqJvoOOfSKAAABLTCCASkGCSqGSIb3DQEHBqCCARowggEWAgEAMIIBDwYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxXCy3KLqyS52zCnWMCARCAgeEluEX/bJ0iVrlzeNF36Ljphp15ybvPCs7096T8JI9Z9f/watuBwOrVGD9y0d+6c2TejHblL3fal/VAvuswS6Rv8RDmuvrIY/VzAkmOvuX3CNYbWUaOzsYxYrxTNC5ozK7DgRejtsAP5nOAWvy8rlik3cBQoRuL23yD5FFlOoKg6cE70CcsQg9wBOkn9ImR0Luh/7+f7RZpFGm7v6adK209ZjRzJ+saUqjZowqbh9D2i6qG/PJmYzjFmSsQAmaOsttZpajSLo14wo8glgE3AxV8LsGWQfYCAdf9/9pp0PrDm+Q=" +} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/list.golden.json b/src/handlers/eval/online-insight/__fixtures__/list.golden.json new file mode 100644 index 000000000..8b0d91f89 --- /dev/null +++ b/src/handlers/eval/online-insight/__fixtures__/list.golden.json @@ -0,0 +1,333 @@ +{ + "onlineEvaluationConfigs": [ + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigId": "ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigName": "ABVfyLatest_ProdEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-17T22:09:59.115Z", + "updatedAt": "2026-06-17T22:10:11.792Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigId": "ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigName": "ABVfyLatest_StagingEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-17T22:09:59.058Z", + "updatedAt": "2026-06-17T22:10:12.095Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyPrerelease_ProdEval-MtuOYB2GZP", + "onlineEvaluationConfigId": "ABVfyPrerelease_ProdEval-MtuOYB2GZP", + "onlineEvaluationConfigName": "ABVfyPrerelease_ProdEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-17T20:28:10.738Z", + "updatedAt": "2026-06-17T20:28:26.527Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyPrerelease_StagingEval-fH26hNBHL6", + "onlineEvaluationConfigId": "ABVfyPrerelease_StagingEval-fH26hNBHL6", + "onlineEvaluationConfigName": "ABVfyPrerelease_StagingEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-17T20:28:09.163Z", + "updatedAt": "2026-06-17T20:28:26.799Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/EvoBugBash_abOnlineEval-Q0Iv1k2cdo", + "onlineEvaluationConfigId": "EvoBugBash_abOnlineEval-Q0Iv1k2cdo", + "onlineEvaluationConfigName": "EvoBugBash_abOnlineEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-12T22:27:27.466Z", + "updatedAt": "2026-06-12T22:27:40.636Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/GwFilterVfy_ctrlEval-T97o5LAtRS", + "onlineEvaluationConfigId": "GwFilterVfy_ctrlEval-T97o5LAtRS", + "onlineEvaluationConfigName": "GwFilterVfy_ctrlEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T18:27:10.496Z", + "updatedAt": "2026-06-15T18:27:26.982Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/GwFilterVfy_treatEval-bTv4u14rUQ", + "onlineEvaluationConfigId": "GwFilterVfy_treatEval-bTv4u14rUQ", + "onlineEvaluationConfigName": "GwFilterVfy_treatEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T18:27:10.370Z", + "updatedAt": "2026-06-15T18:27:27.240Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/OhioTripPlanner_ProdEvalStagin-8wGIBCFzOE", + "onlineEvaluationConfigId": "OhioTripPlanner_ProdEvalStagin-8wGIBCFzOE", + "onlineEvaluationConfigName": "OhioTripPlanner_ProdEvalStagin", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-16T00:09:44.834Z", + "updatedAt": "2026-06-16T00:09:56.722Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/OhioTripPlanner_StagingOnly-A3o7OsFV5a", + "onlineEvaluationConfigId": "OhioTripPlanner_StagingOnly-A3o7OsFV5a", + "onlineEvaluationConfigName": "OhioTripPlanner_StagingOnly", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-16T00:22:22.611Z", + "updatedAt": "2026-06-16T00:22:44.675Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeBundle-UekzTY3B12", + "onlineEvaluationConfigId": "PromoteT2_oeBundle-UekzTY3B12", + "onlineEvaluationConfigName": "PromoteT2_oeBundle", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T17:06:17.370Z", + "updatedAt": "2026-06-15T17:06:45.741Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeCtrl-Vwz1F69rtu", + "onlineEvaluationConfigId": "PromoteT2_oeCtrl-Vwz1F69rtu", + "onlineEvaluationConfigName": "PromoteT2_oeCtrl", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T17:06:17.504Z", + "updatedAt": "2026-06-15T17:06:44.609Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeT2c-2CHeFHBYfA", + "onlineEvaluationConfigId": "PromoteT2_oeT2c-2CHeFHBYfA", + "onlineEvaluationConfigName": "PromoteT2_oeT2c", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T17:06:17.603Z", + "updatedAt": "2026-06-15T17:06:45.174Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeT2t-pKLwdZDXyw", + "onlineEvaluationConfigId": "PromoteT2_oeT2t-pKLwdZDXyw", + "onlineEvaluationConfigName": "PromoteT2_oeT2t", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T17:06:17.784Z", + "updatedAt": "2026-06-15T17:06:45.466Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeTreat-bC8FNM8MEN", + "onlineEvaluationConfigId": "PromoteT2_oeTreat-bC8FNM8MEN", + "onlineEvaluationConfigName": "PromoteT2_oeTreat", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T17:06:24.185Z", + "updatedAt": "2026-06-15T17:06:44.903Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/TEMPAGENTTJAB_MyOnlineEval_prod-n6GVQ2Atnw", + "onlineEvaluationConfigId": "TEMPAGENTTJAB_MyOnlineEval_prod-n6GVQ2Atnw", + "onlineEvaluationConfigName": "TEMPAGENTTJAB_MyOnlineEval_prod", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-05-05T19:22:26.291Z", + "updatedAt": "2026-05-05T19:22:57.162Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/TEMPAGENTTJAB_MyOnlineEval_staging-vPcBdKCji0", + "onlineEvaluationConfigId": "TEMPAGENTTJAB_MyOnlineEval_staging-vPcBdKCji0", + "onlineEvaluationConfigName": "TEMPAGENTTJAB_MyOnlineEval_staging", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-05-05T19:22:33.950Z", + "updatedAt": "2026-05-05T19:22:57.441Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abdemo_evalconfig-ntDb6Q5PLh", + "onlineEvaluationConfigId": "abdemo_evalconfig-ntDb6Q5PLh", + "onlineEvaluationConfigName": "abdemo_evalconfig", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-05-27T22:10:42.270Z", + "updatedAt": "2026-05-27T22:10:51.516Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestdemo_eval_config-Kp7uUyBqZB", + "onlineEvaluationConfigId": "abtestdemo_eval_config-Kp7uUyBqZB", + "onlineEvaluationConfigName": "abtestdemo_eval_config", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-05-26T19:13:52.461Z", + "updatedAt": "2026-05-26T19:14:02.417Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestval_cs_control_eval-seHjGZ7TeH", + "onlineEvaluationConfigId": "abtestval_cs_control_eval-seHjGZ7TeH", + "onlineEvaluationConfigName": "abtestval_cs_control_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-24T22:58:24.637Z", + "updatedAt": "2026-06-24T22:58:38.138Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestval_cs_treatment_eval-AdSjkL4CHu", + "onlineEvaluationConfigId": "abtestval_cs_treatment_eval-AdSjkL4CHu", + "onlineEvaluationConfigName": "abtestval_cs_treatment_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-24T22:58:24.648Z", + "updatedAt": "2026-06-24T22:58:38.393Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ac570c_myonlineeval-6IRBzU3dkS", + "onlineEvaluationConfigId": "ac570c_myonlineeval-6IRBzU3dkS", + "onlineEvaluationConfigName": "ac570c_myonlineeval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-07-06T18:21:13.033Z", + "updatedAt": "2026-07-06T18:21:28.312Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "onlineEvaluationConfigName": "agentcore_cli_online_insight_fixture", + "status": "CREATING", + "executionStatus": "DISABLED", + "createdAt": "2026-08-22T15:09:11.820Z", + "updatedAt": "2026-08-22T15:09:11.820Z", + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ] + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_abtesteval-53zYJX8x4X", + "onlineEvaluationConfigId": "bugbashagent_abtesteval-53zYJX8x4X", + "onlineEvaluationConfigName": "bugbashagent_abtesteval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T18:19:55.627Z", + "updatedAt": "2026-06-15T18:20:08.431Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_lambda_ctrl_eval-EY3GWNCDgH", + "onlineEvaluationConfigId": "bugbashagent_lambda_ctrl_eval-EY3GWNCDgH", + "onlineEvaluationConfigName": "bugbashagent_lambda_ctrl_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T19:41:54.132Z", + "updatedAt": "2026-06-15T19:42:10.231Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_lambda_trtm_eval-IlN5S34Bc5", + "onlineEvaluationConfigId": "bugbashagent_lambda_trtm_eval-IlN5S34Bc5", + "onlineEvaluationConfigName": "bugbashagent_lambda_trtm_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T19:41:54.111Z", + "updatedAt": "2026-06-15T19:42:10.335Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineEval1-QErY9BEWT5", + "onlineEvaluationConfigId": "demoEval_onlineEval1-QErY9BEWT5", + "onlineEvaluationConfigName": "demoEval_onlineEval1", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-07-16T17:26:21.325Z", + "updatedAt": "2026-07-16T17:26:36.933Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineEvalBoto3_20260716180853-tbD9Ob7VUX", + "onlineEvaluationConfigId": "demoEval_onlineEvalBoto3_20260716180853-tbD9Ob7VUX", + "onlineEvaluationConfigName": "demoEval_onlineEvalBoto3_20260716180853", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-07-16T18:08:54.221Z", + "updatedAt": "2026-07-16T18:08:54.477Z", + "description": "boto3-created: onlineEvalBoto3" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineInsights1-QE6As2GTTB", + "onlineEvaluationConfigId": "demoEval_onlineInsights1-QE6As2GTTB", + "onlineEvaluationConfigName": "demoEval_onlineInsights1", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-07-16T17:26:21.625Z", + "updatedAt": "2026-07-16T17:26:37.218Z", + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "clusteringConfig": { + "frequencies": [ + "DAILY" + ] + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineInsightsBoto3_20260716180854-aTLOAI48Jr", + "onlineEvaluationConfigId": "demoEval_onlineInsightsBoto3_20260716180854-aTLOAI48Jr", + "onlineEvaluationConfigName": "demoEval_onlineInsightsBoto3_20260716180854", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-07-16T18:08:54.762Z", + "updatedAt": "2026-07-16T18:08:54.954Z", + "description": "boto3-created: onlineInsightsBoto3", + "insights": [ + { + "insightId": "Builtin.Insight.UserIntent" + } + ], + "clusteringConfig": { + "frequencies": [ + "DAILY" + ] + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/deploytest_myeval-JMv0ihHd30", + "onlineEvaluationConfigId": "deploytest_myeval-JMv0ihHd30", + "onlineEvaluationConfigName": "deploytest_myeval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-04-21T22:47:43.896Z", + "updatedAt": "2026-04-21T22:47:52.237Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/oiVerify1_prodInsights-rPYFuF2WsE", + "onlineEvaluationConfigId": "oiVerify1_prodInsights-rPYFuF2WsE", + "onlineEvaluationConfigName": "oiVerify1_prodInsights", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-08-21T16:49:04.244Z", + "updatedAt": "2026-08-21T16:49:04.485Z", + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ] + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/temp-5Dz9Ox6yMC", + "onlineEvaluationConfigId": "temp-5Dz9Ox6yMC", + "onlineEvaluationConfigName": "temp", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2025-12-02T21:09:17.495Z", + "updatedAt": "2025-12-02T21:09:17.831Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/tjHarnessDerivTest-8rZzgXEHiM", + "onlineEvaluationConfigId": "tjHarnessDerivTest-8rZzgXEHiM", + "onlineEvaluationConfigName": "tjHarnessDerivTest", + "status": "ACTIVE", + "executionStatus": "DISABLED", + "createdAt": "2026-07-27T17:56:04.618Z", + "updatedAt": "2026-07-27T17:56:04.805Z" + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/pause.golden.json b/src/handlers/eval/online-insight/__fixtures__/pause.golden.json new file mode 100644 index 000000000..f9b5382cc --- /dev/null +++ b/src/handlers/eval/online-insight/__fixtures__/pause.golden.json @@ -0,0 +1,7 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "updatedAt": "2026-08-22T15:09:17.878Z", + "status": "UPDATING", + "executionStatus": "DISABLED" +} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/resume.golden.json b/src/handlers/eval/online-insight/__fixtures__/resume.golden.json new file mode 100644 index 000000000..40626d90c --- /dev/null +++ b/src/handlers/eval/online-insight/__fixtures__/resume.golden.json @@ -0,0 +1,7 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-ybMHPI7ui3", + "updatedAt": "2026-08-22T15:09:12.639Z", + "status": "UPDATING", + "executionStatus": "ENABLED" +} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/create/index.tsx b/src/handlers/eval/online-insight/create/index.tsx new file mode 100644 index 000000000..cfd2a6885 --- /dev/null +++ b/src/handlers/eval/online-insight/create/index.tsx @@ -0,0 +1,129 @@ +import z from "zod"; +import type { DataSourceConfig, Filter } from "@aws-sdk/client-bedrock-agentcore-control"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import { SourceResolver, type AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; + +const BUILTIN_INSIGHT_PREFIX = "Builtin.Insight."; +const ARN_PREFIX = "arn:"; + +export const createCreateOnlineInsightHandler = (core: Core, io: AppIO) => + createHandler({ + name: "create", + description: "create an online insight config", + flags: [ + flag("name", "the name of the online insight config", z.string().optional()), + flag("role-arn", "IAM role the online insight assumes", z.string().optional()), + flag("agent", "runtime ID whose traffic to sample", z.string().optional()), + flag( + "endpoint", + "the agent endpoint qualifier to scope monitoring to (default DEFAULT)", + z.string().optional(), + ), + flag( + "data-source-config", + "the traces to evaluate (JSON DataSourceConfig; inline, file://, or - for stdin), as an alternative to --agent", + z.string().optional(), + ), + flag( + "insight", + "insight ID(s) to apply: Builtin.Insight.* identifiers or full ARNs", + z.array(z.string()).optional(), + ), + flag( + "clustering-frequency", + "insight clustering cadence(s): DAILY, WEEKLY, MONTHLY", + z.array(z.enum(["DAILY", "WEEKLY", "MONTHLY"])).optional(), + ), + flag( + "sampling-rate", + "percentage of sessions to sample (0.01-100)", + z.number().min(0.01).max(100).optional(), + ), + flag( + "session-timeout-minutes", + "minutes of inactivity before a session is considered complete (1-1440, default 15)", + z.number().int().min(1).max(1440).optional(), + ), + flag( + "filters", + "trace filters (JSON Filter[]; inline, file://, or - for stdin)", + z.string().optional(), + ), + flag( + "enable-on-create", + "whether to enable evaluation immediately (default true; pass false to create it paused)", + z.enum(["true", "false"]).optional(), + ), + flag( + "description", + "a description of the config's monitoring purpose", + z.string().optional(), + ), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) + throw new InputValidationError("required option '--name ' not specified"); + if (!flags["role-arn"]) + throw new InputValidationError("required option '--role-arn ' not specified"); + if (!flags["sampling-rate"]) + throw new InputValidationError( + "required option '--sampling-rate ' not specified", + ); + if (!flags["insight"] || flags["insight"].length === 0) + throw new InputValidationError("required option '--insight ' not specified"); + + for (const id of flags["insight"]) { + if (!id.startsWith(BUILTIN_INSIGHT_PREFIX) && !id.startsWith(ARN_PREFIX)) + throw new InputValidationError( + `invalid insight "${id}": must be a ${BUILTIN_INSIGHT_PREFIX}* identifier or a full ARN`, + ); + } + + const hasAgent = flags["agent"] !== undefined; + const hasDataSource = flags["data-source-config"] !== undefined; + if (hasAgent === hasDataSource) + throw new InputValidationError( + "specify exactly one of '--agent' or '--data-source-config'", + ); + if (hasDataSource && flags["endpoint"]) + throw new InputValidationError("'--endpoint' can only be used with '--agent'"); + + const source = new SourceResolver({ stdin: io.stdin }); + const frequencies = flags["clustering-frequency"]; + const common = { + name: flags["name"], + description: flags["description"], + samplingRate: flags["sampling-rate"], + sessionTimeoutMinutes: flags["session-timeout-minutes"], + filters: parseJsonFlag( + "filters", + await source.resolveText("filters", flags["filters"]), + ), + insightIds: flags["insight"], + clusteringConfig: frequencies ? { frequencies } : undefined, + evaluationExecutionRoleArn: flags["role-arn"], + enableOnCreate: + flags["enable-on-create"] === undefined + ? undefined + : flags["enable-on-create"] === "true", + }; + + const response = await core.eval.createOnlineInsight( + hasAgent + ? { ...common, agent: flags["agent"]!, endpoint: flags["endpoint"] } + : { + ...common, + dataSourceConfig: parseJsonFlag( + "data-source-config", + await source.resolveText("data-source-config", flags["data-source-config"]), + )!, + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/online-insight/delete/index.tsx b/src/handlers/eval/online-insight/delete/index.tsx new file mode 100644 index 000000000..1572cd7b4 --- /dev/null +++ b/src/handlers/eval/online-insight/delete/index.tsx @@ -0,0 +1,20 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createDeleteOnlineInsightHandler = (core: Core) => + createHandler({ + name: "delete", + description: "delete an online insight config by id", + flags: [flag("id", "the ID of the online insight config to delete", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.deleteOnlineInsight(flags["id"], coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/online-insight/get/index.tsx b/src/handlers/eval/online-insight/get/index.tsx new file mode 100644 index 000000000..dc458c1be --- /dev/null +++ b/src/handlers/eval/online-insight/get/index.tsx @@ -0,0 +1,20 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createGetOnlineInsightHandler = (core: Core) => + createHandler({ + name: "get", + description: "get an online insight config by id", + flags: [flag("id", "the ID of the online insight config", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.getOnlineInsight(flags["id"], coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/online-insight/index.tsx b/src/handlers/eval/online-insight/index.tsx new file mode 100644 index 000000000..3931ac8ae --- /dev/null +++ b/src/handlers/eval/online-insight/index.tsx @@ -0,0 +1,19 @@ +import { Router } from "../../../router"; +import type { AppIO } from "../../../io"; +import type { Core } from "../../types"; +import { createCreateOnlineInsightHandler } from "./create"; +import { createGetOnlineInsightHandler } from "./get"; +import { createListOnlineInsightHandler } from "./list"; +import { createPauseOnlineInsightHandler } from "./pause"; +import { createResumeOnlineInsightHandler } from "./resume"; +import { createDeleteOnlineInsightHandler } from "./delete"; + +export function createOnlineInsightHandler(core: Core, io: AppIO): Router { + return new Router("online-insight", "manage AgentCore online insight configs") + .handler(createCreateOnlineInsightHandler(core, io)) + .handler(createGetOnlineInsightHandler(core)) + .handler(createListOnlineInsightHandler(core)) + .handler(createPauseOnlineInsightHandler(core)) + .handler(createResumeOnlineInsightHandler(core)) + .handler(createDeleteOnlineInsightHandler(core)); +} diff --git a/src/handlers/eval/online-insight/list/index.tsx b/src/handlers/eval/online-insight/list/index.tsx new file mode 100644 index 000000000..284fd21d8 --- /dev/null +++ b/src/handlers/eval/online-insight/list/index.tsx @@ -0,0 +1,23 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createListOnlineInsightHandler = (core: Core) => + createHandler({ + name: "list", + description: "list online insight configs", + flags: [ + flag("next-token", "pagination token returned by a previous request", z.string().optional()), + flag("max-results", "maximum number of items to return", z.number().optional()), + ], + handle: async (ctx, flags) => { + const response = await core.eval.listOnlineInsights( + flags["next-token"], + flags["max-results"], + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/online-insight/online-insight.test.tsx b/src/handlers/eval/online-insight/online-insight.test.tsx new file mode 100644 index 000000000..90602b79f --- /dev/null +++ b/src/handlers/eval/online-insight/online-insight.test.tsx @@ -0,0 +1,341 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { CoreClient } from "../../../core"; +import { + createSilentLogger, + fixtureFactories, + matchGolden, + settle, + TestGlobalConfigAccessor, + testIO, +} from "../../../testing"; +import { createRootHandler } from "../../index"; + +const REGION = "us-west-2"; +const FIXTURES = join(import.meta.dir, "__fixtures__"); + +// Record with RECORD=1 bun test src/handlers/eval/online-insight/online-insight.test.tsx +// The RECORD run creates one online insight config against a real runtime, +// exercises list/get/resume/pause against it, then deletes it, so a recording +// leaves no residue. The id the service assigns is captured from the recorded +// create response, which keeps the dependent fixtures (keyed by request input) +// stable on replay. +const CONFIG_NAME = "agentcore_cli_online_insight_fixture"; + +// online-insight requires an insight id (no evaluators); a builtin needs no setup. +const FIXTURE_INSIGHT_ID = "Builtin.Insight.FailureAnalysis"; + +// The runtime whose traffic the recorded config samples. `--agent` resolves it to +// the CloudWatch log group / service name pair, so the id must exist in the +// fixture account. It is only referenced, never invoked. +const FIXTURE_AGENT_ID = "testAgent_Agent-wm9hYBD93Y"; + +// online-insight has no auto-provisioning, so --role-arn is required. This role +// pre-exists in the fixture account, scoped to online-evaluation-config/*. +const FIXTURE_ROLE_ARN = "arn:aws:iam::725476964917:role/AgentCoreEvalsSDK-us-west-2-a6864eb339"; + +// Well-formed but absent id: it passes the id pattern so the request reaches the +// service and comes back not-found, rather than failing local validation first. +const MISSING_CONFIG_ID = "missing-online-0000000000"; + +function createFixtureCore(): CoreClient { + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); + return new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + createLogsClient, + logger: createSilentLogger(), + }); +} + +// run drives the real router (parsing → middleware → handler → CoreClient) against +// the fixture-backed SDK clients and returns captured stdout. +async function run(args: string[]): Promise { + const io = testIO(); + const root = createRootHandler(createFixtureCore(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + + await root.route(["node", "agentcore", ...args, "--region", REGION]); + return io.stdout(); +} + +// The id assigned by CreateOnlineEvaluationConfig, shared by the tests below. +let configId: string; + +describe("eval online-insight command hierarchy", () => { + test("registers the eval → online-insight command tree", () => { + const root = createRootHandler(createFixtureCore(), { + io: testIO().io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + const onlineInsight = root + .children() + .find((c) => c.name() === "eval") + ?.children() + .find((c) => c.name() === "online-insight"); + + expect(onlineInsight?.children().map((c) => c.name())).toEqual([ + "create", + "get", + "list", + "pause", + "resume", + "delete", + ]); + }); +}); + +describe("online-insight CRUDL", () => { + test("creates an online insight config from an agent", async () => { + const stdout = await run([ + "eval", + "online-insight", + "create", + "--name", + CONFIG_NAME, + "--agent", + FIXTURE_AGENT_ID, + "--role-arn", + FIXTURE_ROLE_ARN, + "--insight", + FIXTURE_INSIGHT_ID, + "--sampling-rate", + "10", + "--enable-on-create", + "false", + ]); + + matchGolden(FIXTURES, "create.golden.json", stdout); + configId = JSON.parse(stdout).onlineEvaluationConfigId; + expect(configId).toBeString(); + }); + + test("lists online insight configs", async () => { + // --json forces the headless path; a bare `list` (no flags) would otherwise + // open the TUI under the empty-invocation middleware. + const stdout = await run(["eval", "online-insight", "list", "--json"]); + + matchGolden(FIXTURES, "list.golden.json", stdout); + expect(JSON.parse(stdout).onlineEvaluationConfigs).toBeArray(); + }); + + test("paginates the list with --max-results and --next-token", async () => { + const firstPage = await run(["eval", "online-insight", "list", "--max-results", "1"]); + matchGolden(FIXTURES, "list-page-1.golden.json", firstPage); + + const first = JSON.parse(firstPage); + expect(first.onlineEvaluationConfigs).toHaveLength(1); + expect(first.nextToken).toBeString(); + + const secondPage = await run([ + "eval", + "online-insight", + "list", + "--max-results", + "1", + "--next-token", + first.nextToken, + ]); + matchGolden(FIXTURES, "list-page-2.golden.json", secondPage); + expect(JSON.parse(secondPage).onlineEvaluationConfigs).toHaveLength(1); + }); + + test("gets the config, exposing the insight, no evaluators, and the custom role", async () => { + const stdout = await run(["eval", "online-insight", "get", "--id", configId]); + matchGolden(FIXTURES, "get.golden.json", stdout); + + const detail = JSON.parse(stdout); + expect(detail.onlineEvaluationConfigName).toBe(CONFIG_NAME); + expect(detail.insights.map((i: { insightId: string }) => i.insightId)).toContain( + FIXTURE_INSIGHT_ID, + ); + // create passed --insight, never --evaluator, so the evaluator list stays empty. + expect(detail.evaluators ?? []).toEqual([]); + // --role-arn is used verbatim; online-insight never provisions a role. + expect(detail.evaluationExecutionRoleArn).toBe(FIXTURE_ROLE_ARN); + }); + + // resume and pause are asserted in one test because the service rejects an + // update while the previous one is still settling (ConflictException, state + // UPDATING). Recording therefore waits for the config to leave UPDATING between + // the two calls; on replay the fixtures are served instantly and the wait is a + // no-op, so the test stays fast and deterministic. + test("resumes then pauses the config, toggling execution status", async () => { + const resumeStdout = await run(["eval", "online-insight", "resume", "--id", configId]); + matchGolden(FIXTURES, "resume.golden.json", resumeStdout); + expect(JSON.parse(resumeStdout).executionStatus).toBe("ENABLED"); + + await settle(); + + const pauseStdout = await run(["eval", "online-insight", "pause", "--id", configId]); + matchGolden(FIXTURES, "pause.golden.json", pauseStdout); + expect(JSON.parse(pauseStdout).executionStatus).toBe("DISABLED"); + // Extended timeout so the settle() wait fits while recording; it is a no-op on replay. + }, 60_000); + + test("deletes the online insight config", async () => { + const stdout = await run(["eval", "online-insight", "delete", "--id", configId]); + matchGolden(FIXTURES, "delete.golden.json", stdout); + }); + + test("propagates ResourceNotFoundException from get", async () => { + await expect( + run(["eval", "online-insight", "get", "--id", MISSING_CONFIG_ID]), + ).rejects.toMatchObject({ name: "ResourceNotFoundException" }); + }); +}); + +// Flag parsing never reaches the SDK, so these need no fixtures. +describe("flag validation", () => { + test("create requires --role-arn", async () => { + await expect( + run([ + "eval", + "online-insight", + "create", + "--name", + CONFIG_NAME, + "--agent", + FIXTURE_AGENT_ID, + "--insight", + FIXTURE_INSIGHT_ID, + "--sampling-rate", + "10", + ]), + ).rejects.toThrow("required option '--role-arn ' not specified"); + }); + + test("create requires --insight", async () => { + await expect( + run([ + "eval", + "online-insight", + "create", + "--name", + CONFIG_NAME, + "--agent", + FIXTURE_AGENT_ID, + "--role-arn", + FIXTURE_ROLE_ARN, + "--sampling-rate", + "10", + ]), + ).rejects.toThrow("required option '--insight ' not specified"); + }); + + test("create rejects an insight id that is neither a builtin nor an ARN", async () => { + await expect( + run([ + "eval", + "online-insight", + "create", + "--name", + CONFIG_NAME, + "--agent", + FIXTURE_AGENT_ID, + "--role-arn", + FIXTURE_ROLE_ARN, + "--insight", + "nope", + "--sampling-rate", + "10", + ]), + ).rejects.toThrow(/invalid insight "nope"/); + }); + + test("create requires exactly one of --agent or --data-source-config", async () => { + await expect( + run([ + "eval", + "online-insight", + "create", + "--name", + CONFIG_NAME, + "--role-arn", + FIXTURE_ROLE_ARN, + "--insight", + FIXTURE_INSIGHT_ID, + "--sampling-rate", + "10", + ]), + ).rejects.toThrow(/exactly one of '--agent' or '--data-source-config'/); + }); + + test("create rejects both --agent and --data-source-config", async () => { + await expect( + run([ + "eval", + "online-insight", + "create", + "--name", + CONFIG_NAME, + "--agent", + FIXTURE_AGENT_ID, + "--data-source-config", + '{"cloudWatchLogs":{"logGroupNames":["/custom"],"serviceNames":["svc"]}}', + "--role-arn", + FIXTURE_ROLE_ARN, + "--insight", + FIXTURE_INSIGHT_ID, + "--sampling-rate", + "10", + ]), + ).rejects.toThrow(/exactly one of '--agent' or '--data-source-config'/); + }); + + test("create rejects --endpoint without --agent", async () => { + await expect( + run([ + "eval", + "online-insight", + "create", + "--name", + CONFIG_NAME, + "--data-source-config", + '{"cloudWatchLogs":{"logGroupNames":["/custom"],"serviceNames":["svc"]}}', + "--endpoint", + "prod", + "--role-arn", + FIXTURE_ROLE_ARN, + "--insight", + FIXTURE_INSIGHT_ID, + "--sampling-rate", + "10", + ]), + ).rejects.toThrow(/'--endpoint' can only be used with '--agent'/); + }); + + test("create rejects malformed --data-source-config JSON", async () => { + await expect( + run([ + "eval", + "online-insight", + "create", + "--name", + CONFIG_NAME, + "--data-source-config", + "not-json", + "--role-arn", + FIXTURE_ROLE_ARN, + "--insight", + FIXTURE_INSIGHT_ID, + "--sampling-rate", + "10", + ]), + ).rejects.toThrow(/Invalid JSON for option '--data-source-config'/); + }); + + // --json forces the headless path so the required-flag error surfaces; without + // it a bare invocation opens the TUI under the empty-invocation middleware. + test.each(["get", "pause", "resume", "delete"])("%s requires --id", async (command) => { + await expect(run(["eval", "online-insight", command, "--json"])).rejects.toThrow( + /required option '--id ' not specified/, + ); + }); +}); diff --git a/src/handlers/eval/online-insight/pause/index.tsx b/src/handlers/eval/online-insight/pause/index.tsx new file mode 100644 index 000000000..7c28ca54f --- /dev/null +++ b/src/handlers/eval/online-insight/pause/index.tsx @@ -0,0 +1,26 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createPauseOnlineInsightHandler = (core: Core) => + createHandler({ + name: "pause", + description: "pause an online insight config", + flags: [flag("id", "the ID of the online insight config to pause", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson( + await core.eval.setOnlineInsightExecutionStatus( + flags["id"], + "DISABLED", + coreOptsFromCtx(ctx), + ), + ); + }, + }); diff --git a/src/handlers/eval/online-insight/resume/index.tsx b/src/handlers/eval/online-insight/resume/index.tsx new file mode 100644 index 000000000..985738709 --- /dev/null +++ b/src/handlers/eval/online-insight/resume/index.tsx @@ -0,0 +1,26 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createResumeOnlineInsightHandler = (core: Core) => + createHandler({ + name: "resume", + description: "resume a paused online insight config", + flags: [flag("id", "the ID of the online insight config to resume", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson( + await core.eval.setOnlineInsightExecutionStatus( + flags["id"], + "ENABLED", + coreOptsFromCtx(ctx), + ), + ); + }, + }); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 226c17f81..50340aa6f 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -146,6 +146,21 @@ export type CreateOnlineEvalInput = { | { agent?: undefined; endpoint?: undefined; dataSourceConfig: DataSourceConfig } ); +export type CreateOnlineInsightInput = { + name: string; + description?: string; + samplingRate: number; + sessionTimeoutMinutes?: number; + filters?: Rule["filters"]; + insightIds: string[]; + clusteringConfig?: { frequencies: ("DAILY" | "WEEKLY" | "MONTHLY")[] }; + evaluationExecutionRoleArn: string; + enableOnCreate?: boolean; +} & ( + | { agent: string; endpoint?: string; dataSourceConfig?: undefined } + | { agent?: undefined; endpoint?: undefined; dataSourceConfig: DataSourceConfig } +); + // UpdateOnlineEvalInput carries the fields a caller may change on an online // evaluation config. Undefined fields are left untouched by Core (merged over // the current config, since UpdateOnlineEvaluationConfig replaces the whole @@ -344,6 +359,26 @@ export interface CoreEvalClient { options: CoreOptions, ): Promise; + createOnlineInsight( + input: CreateOnlineInsightInput, + options: CoreOptions, + ): Promise; + getOnlineInsight(id: string, options: CoreOptions): Promise; + listOnlineInsights( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise; + setOnlineInsightExecutionStatus( + id: string, + executionStatus: "ENABLED" | "DISABLED", + options: CoreOptions, + ): Promise; + deleteOnlineInsight( + id: string, + options: CoreOptions, + ): Promise; + createConfigurationBundle( input: CreateConfigurationBundleInput, options: CoreOptions, diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 50047db83..a4495da0c 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -129,6 +129,7 @@ import type { CreateConfigurationBundleInput, CreateDatasetInput, CreateOnlineEvalInput, + CreateOnlineInsightInput, DatasetUpdateResult, DatasetUpdateProgressEvent, EvaluateInput, @@ -1787,6 +1788,56 @@ export class TestEvalClient implements CoreEvalClient { return this.onlineEvalDeleteResponse; } + async createOnlineInsight( + input: CreateOnlineInsightInput, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "createOnlineInsight", args: [input, options] }); + if (this.error) throw this.error; + return this.onlineEvalCreateResponse; + } + + async getOnlineInsight( + id: string, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "getOnlineInsight", args: [id, options] }); + if (this.error) throw this.error; + return this.onlineEvalGetResponse; + } + + async listOnlineInsights( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "listOnlineInsights", args: [nextToken, maxResults, options] }); + if (this.error) throw this.error; + return this.onlineEvalListResponses.get(nextToken) ?? { onlineEvaluationConfigs: [] }; + } + + async setOnlineInsightExecutionStatus( + id: string, + executionStatus: "ENABLED" | "DISABLED", + options: CoreOptions, + ): Promise { + this.calls.push({ + method: "setOnlineInsightExecutionStatus", + args: [id, executionStatus, options], + }); + if (this.error) throw this.error; + return this.onlineEvalUpdateResponse; + } + + async deleteOnlineInsight( + id: string, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "deleteOnlineInsight", args: [id, options] }); + if (this.error) throw this.error; + return this.onlineEvalDeleteResponse; + } + async createConfigurationBundle( input: CreateConfigurationBundleInput, options: CoreOptions,