From 6934cf9b8a53113b822596d465fce7d8a9e43c76 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack Date: Tue, 23 Jun 2026 21:54:55 +0530 Subject: [PATCH 01/33] =?UTF-8?q?feat(rca):=20scaffold=20plugin=20?= =?UTF-8?q?=E2=80=94=20manifest,=20MCP=20wiring,=20config,=20command,=20RE?= =?UTF-8?q?ADME?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Identity-only .claude-plugin/plugin.json; root .mcp.json wires the bstack MCP server (stdio); config/rca.config.json centralizes all formerly-hardcoded product/infra values (no kubectl/chitragupta/bifrost literals); /rca-build command parses build id + mode and hands off to the skill. Co-Authored-By: Claude Opus 4.8 --- .claude-plugin/plugin.json | 11 +++++++ .env.example | 8 +++++ .gitignore | 4 +++ .mcp.json | 14 ++++++++ README.md | 67 ++++++++++++++++++++++++++++++++++++-- commands/rca-build.md | 34 +++++++++++++++++++ config/rca.config.json | 25 ++++++++++++++ package.json | 10 ++++++ 8 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 .claude-plugin/plugin.json create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 .mcp.json create mode 100644 commands/rca-build.md create mode 100644 config/rca.config.json create mode 100644 package.json diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..c8c4beb --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,11 @@ +{ + "name": "tfa-rca", + "description": "Drive collaborative root-cause analysis over all failed tests of a build, generic across product and infra.", + "version": "0.1.0", + "author": { + "name": "BrowserStack", + "url": "https://www.browserstack.com" + }, + "homepage": "https://github.com/browserstack/browserstack-ai-tfa-demo", + "license": "MIT" +} diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d86819e --- /dev/null +++ b/.env.example @@ -0,0 +1,8 @@ +# BrowserStack credentials — used by the bundled bstack MCP server for +# listTestIds + tfaRcaTurn. Per-user; never commit real values. +BROWSERSTACK_USERNAME= +BROWSERSTACK_ACCESS_KEY= + +# Observability base URL the TFA RCA chat runs against. Optional — +# the bstack MCP server defaults to its rengg-tfa staging URL when unset. +# O11Y_TFA_RCA_BASE_URL=https://api-observability-rengg-tfa.bsstag.com diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9045f9d --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +.env +# Per-run RCA batch state (the CSV/WAL spine + report) is workspace-local. +.rca/ diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..0502929 --- /dev/null +++ b/.mcp.json @@ -0,0 +1,14 @@ +{ + "mcpServers": { + "bstack": { + "type": "stdio", + "command": "npx", + "args": ["-y", "@browserstack/mcp-server"], + "env": { + "BROWSERSTACK_USERNAME": "${BROWSERSTACK_USERNAME}", + "BROWSERSTACK_ACCESS_KEY": "${BROWSERSTACK_ACCESS_KEY}", + "O11Y_TFA_RCA_BASE_URL": "${O11Y_TFA_RCA_BASE_URL}" + } + } + } +} diff --git a/README.md b/README.md index 423d780..ff148ab 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,65 @@ -# browserstack-ai-tfa-demo -AI TFA Demo +# tfa-rca — generic multi-client RCA agent plugin + +Drive BrowserStack's collaborative root-cause-analysis loop over **all failed +tests of a build**, generic across product and infra, from inside an agentic +MCP client (Claude Code / Cursor / Codex). + +The plugin wraps two stable MCP tools — `listTestIds` and `tfaRcaTurn` (from the +`bstack` MCP server) — and adds the harness that batches RCA over a whole build, +clusters failures by signature, routes evidence requests to whatever +skills/tools the client already has, and writes a per-test RCA into the TRA +dashboard. + +> It **discovers and delegates** to the infra skills/tools already in your +> client (GitHub, k8s/EKS, kibana/other logs, metrics). It does **not** install +> or own those connectors. + +## Install + +```bash +git clone https://github.com/browserstack/browserstack-ai-tfa-demo.git +cd browserstack-ai-tfa-demo +cp .env.example .env # fill in BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY +claude --plugin-dir ./ +``` + +The plugin auto-configures on load: the `bstack` MCP server (from `.mcp.json`), +the `/rca-build` command, the `rca-build` skill, and the `ai-tfa-coordinator` +agent are all discovered by convention. + +## Usage + +``` +/rca-build +/rca-build build_id= mode=auto +``` + +On start the plugin runs a **mandatory pre-flight intake** asking for your +product + automation repos, working branch, default branch, and the PRs in +play, plus the build id. Every question is answerable with "I don't have one" → +the run proceeds RCA-only. + +## Modes + +- **auto** — a dynamic workflow drives the whole batch (5 tests concurrent), no + mid-run prompts. When evidence can't be gathered (no matching skill), it + reports "unavailable" back to the TFA agent, which finalizes best-effort. +- **interactive** — the main session spawns subagents (5 at a time); on an + evidence gap a subagent returns the gap to the main agent, which asks you, + then feeds the answer back. + +`auto` means autonomy *during* the batch from an interactive session — not +headless. Running `claude -p` with a required input missing ends immediately. + +## Requirements + +- The `bstack` MCP server (bundled via `.mcp.json`). +- Credentials in `.env` (or your client's MCP env). +- For full evidence coverage: whatever GitHub / infra / logging / metrics + skills your client already has. Missing ones degrade gracefully (the RCA's + confidence band reflects what evidence was actually available). + +## Layout + +See `docs/plans/2026-06-23-001-feat-generic-rca-agent-plugin-plan.md` for the +implementation plan and `docs/brainstorms/` for the requirements. diff --git a/commands/rca-build.md b/commands/rca-build.md new file mode 100644 index 0000000..7a7a829 --- /dev/null +++ b/commands/rca-build.md @@ -0,0 +1,34 @@ +--- +description: Run collaborative RCA over all failed tests of a BrowserStack build +--- + +# /rca-build + +Entry point for the generic RCA harness. Drives a collaborative root-cause +analysis loop over **every failed test** of a build, generic across product and +infra. + +## Input + +`$ARGUMENTS` carries the build id (and optional flags). Accepted forms: + +- bare build id: `qzqhbfa5bkjakcbxtvy2siwtpcvsvgm9fxfyb03d5` +- `build_id=` +- a build dashboard link (the id is extracted) +- optional `mode=auto` | `mode=interactive` (default: prompt the user) + +Parse the build id. If none is present, this is a required input: + +- in an interactive session → ask the user for it +- in headless (`claude -p`) → **end immediately** (fail fast), do not hang + +## Behavior + +Invoke the `rca-build` skill, passing the parsed build id and mode. The skill +owns the full flow: mandatory pre-flight GitHub intake → discovery via +`listTestIds` → CSV/WAL spine → failure-signature clustering → fan-out +(auto = dynamic workflow / interactive = subagents) → per-test RCA loop via +`tfaRcaTurn` → report. + +Do not re-implement the orchestration here — this command only parses input and +hands off to the skill. diff --git a/config/rca.config.json b/config/rca.config.json new file mode 100644 index 0000000..b7633bb --- /dev/null +++ b/config/rca.config.json @@ -0,0 +1,25 @@ +{ + "$comment": "Central config for the generic RCA harness. All formerly-hardcoded product/infra values live here. No kubectl/chitragupta/bifrost literals — infra tools are discovered at runtime via the capability manifest (see skills/rca-build/references/evidence-routing.md).", + "mcpServerName": "bstack", + "concurrency": 5, + "turnCap": 6, + "turnMessageMaxChars": 5000, + "pollSoftPendingMs": 90000, + "reaperHeartbeatTtlSec": 600, + "errorSummaryMaxChars": 200, + "paths": { + "stateDir": ".rca", + "csvFile": ".rca/rca-state.csv", + "reportFile": ".rca/rca-report.md" + }, + "evidenceRouting": { + "test_logs": { "owner": "tfa", "skip": true }, + "product_code": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, + "deploy": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, + "ci": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, + "k8s": { "capability": "k8s", "discoveryHints": [] }, + "kibana": { "capability": "logs", "discoveryHints": [] }, + "metrics": { "capability": "metrics", "discoveryHints": [] }, + "other": { "capability": "other", "discoveryHints": [] } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..c11e40f --- /dev/null +++ b/package.json @@ -0,0 +1,10 @@ +{ + "name": "tfa-rca-plugin", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "Generic multi-client RCA agent plugin harness", + "scripts": { + "test": "node --test tests/" + } +} From f0d5cf63e017a5669764a7f59446d16a59550d0b Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack Date: Tue, 23 Jun 2026 21:58:47 +0530 Subject: [PATCH 02/33] feat(rca): generic per-test RCA coordinator + evidence-routing registry Port the obs-tfa-rca loop decoupled: ai-tfa-coordinator drives tfaRcaTurn to a terminal RCA (turn-cap, one-thread, soft-PENDING, digest-not-dump) with the gather mechanism routed by capability (no kubectl/chitragupta/bifrost literals). lib/routing.mjs classifies each ask skip/gather/gap against the config registry + capability manifest; the gap action is the only mode fork (auto=unavailable, interactive=ask-user). references/evidence-routing.md carries the digest format and size caps verbatim. Adds sibling pre-seed one-turn-confirm hook. Co-Authored-By: Claude Opus 4.8 --- agents/ai-tfa-coordinator.md | 185 ++++++++++++++++++ lib/routing.mjs | 75 +++++++ package.json | 2 +- .../rca-build/references/evidence-routing.md | 133 +++++++++++++ tests/routing.test.mjs | 80 ++++++++ 5 files changed, 474 insertions(+), 1 deletion(-) create mode 100644 agents/ai-tfa-coordinator.md create mode 100644 lib/routing.mjs create mode 100644 skills/rca-build/references/evidence-routing.md create mode 100644 tests/routing.test.mjs diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md new file mode 100644 index 0000000..e2045fb --- /dev/null +++ b/agents/ai-tfa-coordinator.md @@ -0,0 +1,185 @@ +--- +name: ai-tfa-coordinator +description: 'Per-test collaborative-RCA coordinator. Given ONE testRunId, drives the tfaRcaTurn MCP loop to a terminal root cause: TFA reads the run logs; this coordinator supplies every non-log evidence ask (product code, k8s, kibana, metrics, deploy, ci) using whatever skills/tools the client has, routed through the capability manifest. Skips every test_logs ask (TFA owns logs). Emits a structured RCA_OUTPUT block. Generic over product and infra — no hardcoded tools. Examples: +- orchestrator: Agent(subagent_type="ai-tfa-coordinator", prompt="RCA testRunId=39 — error: empty buildName rejected on POST /builds") → drives the loop, returns RCA_OUTPUT +- sibling confirm: Agent(subagent_type="ai-tfa-coordinator", prompt="RCA testRunId=40 — pre-seed: cause=, suspect PR=#7421") → one-turn confirm against this test logs +- user: "run collaborative RCA on test run 39" → single-test loop to RESOLVED/BLOCKED/PENDING' +tools: [Bash, Read, Grep, Glob, Task, mcp__*__tfaRcaTurn, mcp__github__*] +model: sonnet +--- + +# Per-Test Collaborative RCA Coordinator (`ai-tfa-coordinator`) + +Drives the `tfaRcaTurn` MCP loop for a **single** failed test to a terminal RCA. +The collaboration contract is fixed: **TFA owns logs; this coordinator owns +everything else.** TFA (server-side, via the tool) reads the run's logs from its +own access and emits typed evidence asks; this coordinator fulfills every +**non-log** ask using whatever skills/tools the client has — routed through the +capability manifest — digests the findings, and feeds them back on the same +thread until TFA converges. TFA authors the RCA into the TRA dashboard. + +This coordinator is the **reusable unit**: it takes one `testRunId` and runs +standalone, driven by the auto workflow, an interactive subagent, or a thin +sequential harness. It is **generic over product and infra** — it names no +`kubectl` / `chitragupta` / `bifrost`; it routes by *capability*. + +## Inputs + +- `testRunId` — **required**, the integer test-run ID. Maps to the tool's `testRunId` arg. +- `error_digest` — optional short error title + endpoint (NOT logs) for the first-turn message. +- `pre_seed` — optional. For a **cluster sibling**: the representative's + `root_cause` + suspect `related_prs`. When present, the first-turn message + states the hypothesis and asks TFA to **confirm it against this test's own logs**. +- `resume` — optional `{ threadId, turnId }` from a prior PENDING run. +- `manifest` — the capability manifest `{ capability: { available, via } }` (from the orchestrator's pre-compute). +- `mode` — `auto` | `interactive`. Selects the **gap-resolver** (see below). + +If `testRunId` is missing or not parseable as an integer, emit a `failed` +`RCA_OUTPUT` block with `root_cause: "no testRunId provided"` and stop — do not +call the tool. + +## Operating principles + +1. **Logs by TFA — the core contract.** Never seed logs in the first turn; + **skip every ask with `evidenceType === "test_logs"`**. Never fetch, paste, or + digest log content. Logs are TFA's job. +2. **Read-only.** Every gather mechanism is read-only. Never write to a repo, + cluster, ticket, or the run. Produce a block and stop. +3. **Turn-cap** = `turnCap` from `config/rca.config.json` (default 6). If the cap + is hit while still `NEEDS_INFO`, end as `PENDING` (note `turn-cap`) — never an + extra turn, never a busy-wait. +4. **One thread per test.** First turn omits `threadId`; capture it from the + response and reuse it on every follow-up. Never start a second thread. +5. **Soft-PENDING ends the loop.** A tool result of `status: "PENDING"` (in-call + poll exceeded its wall-clock cap) ends the loop immediately as `PENDING`, + carrying `threadId` + `turnId` for a later resume. Do not re-poll or sleep. +6. **Digest, don't dump.** Every follow-up `message` carries digested findings + (`ask → found → snippet/link`), never raw log tails, full diffs, or full files. + Size caps + block shape live in `references/evidence-routing.md` — read it + before fulfilling any ask. The tool caps `message` at 5000 chars. +7. **Report gaps, don't drop them.** An ask the coordinator cannot fulfill becomes + a `not-found` / `unreachable` / `unavailable` block, never a silent omission. +8. **Never editorialize.** Report findings (suspect PR, server-side error line), + not verdicts. The root cause is TFA's to state on `RESOLVED`; pass its `rca` + through verbatim. + +## The gap-resolver (mode fork) + +Routing an ask yields `skip` / `gather` / `gap` (see `references/evidence-routing.md`). +The only behavioral difference between modes is what happens on a **gap** (no +capability available for that `evidenceType`): + +- **auto** → emit an `unavailable` block back to TFA (no user prompt). TFA + finalizes best-effort with lower confidence. +- **interactive** → **return the gap to the caller** (the main agent), which asks + the user (A1) for that data, then feeds the answer back. A subagent cannot + prompt the user itself. + +Everything else — the loop, routing, digest, caps, output — is identical across +modes. Do not fork the loop; only the gap action differs. + +## The loop + +``` +0. Parse inputs → testRunId (int). Build the first-turn DIGEST: + - pre_seed present → "Hypothesis from cluster representative: . + Suspect PR(s): . Confirm against THIS test's logs." (NO logs) + - error_digest present → "Error: " (NO logs, NO threadId) + - neither → "Initiating collaborative RCA for test run <id>." +1. SUBMIT turn 1: tfaRcaTurn(testRunId=<id>, message=<digest>). Capture threadId. turns_used = 1. + (resume case: tfaRcaTurn(testRunId, threadId, turnId) instead, then continue at 2.) +2. CLASSIFY result.status: + RESOLVED → capture rca; END (RESOLVED). + BLOCKED → capture reason + unmetAsks; END (BLOCKED). + PENDING → capture threadId + turnId; END (PENDING, note "soft-pending"). + NEEDS_INFO → go to 3. +3. ROUTE the asks (read references/evidence-routing.md; route via lib/routing.mjs): + For each ask, high → medium → low: + skip → record in asks_skipped, emit nothing. + gather → run the discovered skill/tool for its capability, digest into one block. + Record evidenceType in asks_fulfilled (dedupe). + gap → run the mode's gap-resolver (auto: unavailable block; interactive: return to caller). + Concatenate per-ask blocks into the next-turn MESSAGE (respect size caps). +4. SUBMIT follow-up on the SAME thread: tfaRcaTurn(testRunId, message, threadId). turns_used += 1. +5. TURN-CAP CHECK: if turns_used >= turnCap and still NEEDS_INFO → END (PENDING, "turn-cap"). + else → go to 2 with the new result. +6. EMIT the RCA_OUTPUT block from the captured terminal state. +``` + +**Sibling confirm (cluster member).** When `pre_seed` is present the first turn +states the representative's hypothesis and asks TFA to confirm against this +test's own logs. If TFA `RESOLVED`s in one turn → a logs-grounded per-test RCA at +minimal cost. If TFA instead returns `NEEDS_INFO` / `BLOCKED` (the hypothesis +does not hold for this test), **fall back to the normal loop** — never blindly +inherit the representative's cause. + +## Output contract — `RCA_OUTPUT` + +Emit **exactly one** block at the end of every run (including the `failed` +no-input case). The orchestrator parses it into one CSV row / report record. + +``` +RCA_OUTPUT_START + +## testRunId +<integer> + +## status +<RESOLVED | BLOCKED | PENDING | failed> + +## confidence +<high | medium | low | unknown> # from the terminal turn; unknown for PENDING/failed + +## root_cause +<RESOLVED → rca.root_cause verbatim · BLOCKED → TFA's reason · PENDING/failed → "not available" or the note> + +## possible_fix +<RESOLVED → rca.possible_fix verbatim · else "not available"> + +## related_prs +- <each PR TFA recorded in rca.related_prs; "none" if empty> + +## suspect_signals +- <each non-log signal surfaced: suspect PR / deploy / server-side error line; "none" if empty> + +## thread_id +<threadId from the first turn · "not available" if none> + +## turn_id +<turnId — present for PENDING (resume handle); else "not available"> + +## turns_used +<integer 1..turnCap> + +## asks_fulfilled +- <evidenceType> # every non-test_logs type fulfilled; "none" if empty + +## asks_skipped +- test_logs # present once a test_logs ask appeared + +## asks_unavailable +- <evidenceType> # gaps with no capability (drives the coverage stamp, U10); "none" if empty + +RCA_OUTPUT_END +``` + +Notes: +- `status` is one of exactly four values. `turn-cap` and `soft-pending` both + report as `PENDING`; note which in `root_cause`. +- `asks_skipped` always includes `test_logs` whenever TFA asked for logs. + `asks_fulfilled` **never** includes `test_logs`. +- `asks_unavailable` is the evidence-coverage signal U10 turns into a confidence band. +- `failed` is the no-parseable-result / no-input case; the orchestrator + synthesizes a `failed` row if this coordinator dies — keep the block valid. + +## Hard limits + +- **Never** fulfill or seed a `test_logs` ask — TFA owns logs. +- **Never** exceed `turnCap` `tfaRcaTurn` calls in one run. +- **Never** start a second thread for the same test — reuse the first turn's `threadId`. +- **Never** busy-wait / re-poll on a soft-`PENDING` — end and report it resumable. +- **Never** dump raw logs, full diffs, or full file contents into a turn message — digest only. +- **Never** write to any repo / cluster / ticket / the run — every action is read-only. +- **Never** editorialize a cause — pass TFA's `rca` through verbatim. +- **Never** blindly inherit a representative's cause for a sibling — confirm against its own logs. +- **Always** emit exactly one valid `RCA_OUTPUT` block, even on the `failed` path. diff --git a/lib/routing.mjs b/lib/routing.mjs new file mode 100644 index 0000000..291738e --- /dev/null +++ b/lib/routing.mjs @@ -0,0 +1,75 @@ +// Evidence-routing registry (D3). Maps a TFA `ask.evidenceType` onto an +// action, given the run's capability manifest. Pure + dependency-free so it is +// testable and reusable by both the auto workflow and interactive subagents. +// +// `test_logs` is the TFA agent's own evidence and is always skipped. Every +// other type routes to a capability; whether that capability is *available* is +// decided by the manifest (built once per run — see U6 / buildManifest). + +import { readFileSync } from "node:fs"; + +export const TEST_LOGS = "test_logs"; + +const PRIORITY_RANK = { high: 0, medium: 1, low: 2 }; + +// Load and parse config/rca.config.json from an absolute or cwd-relative path. +export function loadConfig(configPath) { + return JSON.parse(readFileSync(configPath, "utf8")); +} + +// Order a turn's asks high → medium → low (unknown priority sorts last). +export function orderAsks(asks = []) { + return [...asks].sort( + (a, b) => + (PRIORITY_RANK[a?.priority] ?? 99) - (PRIORITY_RANK[b?.priority] ?? 99), + ); +} + +// Classify one ask. Returns one of: +// { action: "skip", ... } — test_logs / TFA-owned; the coordinator emits nothing +// { action: "gather", ... } — a capability is available; gather + digest +// { action: "gap", ... } — no capability; the caller's resolveGap() decides +// (auto → "unavailable" block; interactive → ask the user) +// +// `manifest` shape: { [capability]: { available: boolean, via?: string } }. +export function routeAsk(ask, config, manifest = {}) { + const evidenceType = ask?.evidenceType ?? "other"; + const routing = config?.evidenceRouting ?? {}; + const entry = routing[evidenceType] ?? routing.other ?? { capability: "other" }; + + if (entry.skip || entry.owner === "tfa") { + return { evidenceType, action: "skip", reason: "tfa-owned" }; + } + + const capability = entry.capability ?? "other"; + const cap = manifest[capability]; + if (cap && cap.available) { + return { + evidenceType, + action: "gather", + capability, + via: cap.via ?? null, + }; + } + + return { + evidenceType, + action: "gap", + capability, + discoveryHints: entry.discoveryHints ?? [], + reason: "no-capability", + }; +} + +// Split a turn's asks into the three buckets, in priority order. The +// coordinator gathers `gather`, runs resolveGap() on each `gap`, and records +// `skip` (test_logs) without emitting anything. +export function routeAsks(asks, config, manifest = {}) { + const ordered = orderAsks(asks); + const buckets = { skip: [], gather: [], gap: [] }; + for (const ask of ordered) { + const routed = routeAsk(ask, config, manifest); + buckets[routed.action].push({ ask, ...routed }); + } + return buckets; +} diff --git a/package.json b/package.json index c11e40f..27344a7 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,6 @@ "type": "module", "description": "Generic multi-client RCA agent plugin harness", "scripts": { - "test": "node --test tests/" + "test": "node --test" } } diff --git a/skills/rca-build/references/evidence-routing.md b/skills/rca-build/references/evidence-routing.md new file mode 100644 index 0000000..e6cc4d0 --- /dev/null +++ b/skills/rca-build/references/evidence-routing.md @@ -0,0 +1,133 @@ +# Evidence Routing + +Load this file **before fulfilling any `NEEDS_INFO` ask** in the per-test RCA +loop (`agents/ai-tfa-coordinator`). It maps each TFA `evidenceType` to a +**capability** (not a hardcoded tool), and defines the **digest** the coordinator +submits on the next turn. + +The core contract: **TFA owns logs; the client agent owns everything else.** The +coordinator never seeds logs and never fulfills a `test_logs` ask. Every other +`evidenceType` routes to a capability that is gathered via **whatever skill/tool +the client actually has** for it (discovered once into the capability manifest — +see `SKILL.md` § Pre-compute). There are **no `kubectl` / `chitragupta` / +`bifrost` literals here** — that is the whole point of going generic. + +The registry logic lives in `lib/routing.mjs` (`routeAsk` / `routeAsks`); this +file is the human/agent-facing contract for the digest and the size caps. + +--- + +## How a turn's asks are processed + +A `NEEDS_INFO` turn returns `asks: TfaAsk[]`, each `{ what, why, evidenceType, +priority }`. For each ask, in descending `priority` (`high` → `medium` → `low`): + +1. Route the `evidenceType` (via `lib/routing.mjs` → the config registry + + capability manifest). The result is one of three actions: + - **skip** — `test_logs` (TFA-owned). Gather nothing; record in `asks_skipped`. + - **gather** — a capability is available. Run its discovered skill/tool scoped + by `what` / `why`, then digest the result into one ask block. + - **gap** — no capability is available. Hand the ask to the injected + **`resolveGap()`** policy: + - **auto mode** → emit an `unavailable` block back to TFA (no user prompt). + - **interactive mode** → return the gap to the main agent, which asks the + user, then feeds the answer back. +2. Concatenate the per-ask blocks into the next-turn `message` and resubmit on + the same `threadId`. + +An ask that cannot be fulfilled is **never silently dropped** — it becomes a +`not-found` / `unreachable` / `unavailable` block so TFA can reason about the gap. + +--- + +## Routing table (capability, not tool) + +`evidenceType` literals are exactly those `tfaRcaTurn` emits: `test_logs`, +`product_code`, `k8s`, `kibana`, `metrics`, `deploy`, `ci`, `other`. + +| `evidenceType` | Capability | Gathered via (discovered at runtime) | +|---|---|---| +| `test_logs` | — (TFA, skip) | never gathered; TFA self-serves from its own log access | +| `product_code` | `github` | the client's GitHub capability — **GitHub MCP if present, else `gh`** (see `references/github-evidence.md`) | +| `deploy` | `github` | deploy timeline via the GitHub capability (releases/tags + deploy record) | +| `ci` | `github` | CI config + run history via the GitHub capability | +| `k8s` | `k8s` | whatever k8s/EKS skill the client has — discovered, not assumed | +| `kibana` | `logs` | whatever log-search skill the client has (kibana or other) | +| `metrics` | `metrics` | whatever metrics skill the client has | +| `other` | `other` | best-effort by ask text; else a `not-found` block | + +The mapping is data in `config/rca.config.json` (`evidenceRouting`), so a +different deployment can remap `evidenceType → capability` without code changes. + +**Deployment-state guard:** a suspect PR only matters if its code was actually +live in the run's env at the failure window. If you can cheaply confirm it was +not deployed / behind an OFF flag, say so in the digest rather than feeding TFA a +suspect that could not have caused the failure. (Full protocol: U9 / +`references/github-evidence.md`.) + +--- + +## Digest format + +The single most important discipline: **digested input, not raw dumps.** Every +turn's `message` loads into the agent's context *and* is sent to TFA; a raw log +tail or full PR diff blows both budgets and degrades TFA's reasoning. Supply the +*findings*, not the *haystack*. + +### Per-ask block shape — `ask → found → snippet/link` + +``` +ASK: <verbatim `what` from the TfaAsk, ≤ 120 chars> +TYPE: <evidenceType> +FOUND: <yes | no | partial> +SUMMARY: <1–3 sentences — the finding, in the agent's words. ≤ 400 chars> +SNIPPET: + <the load-bearing excerpt only — see size caps. Omit if a LINK fully carries it.> +LINK: <permalink to the source — PR/commit/log-search/metrics panel/deploy record. Omit if N/A.> +``` + +- `SUMMARY` is the answer. `SNIPPET` is the *minimum* evidence backing it. `LINK` + lets TFA (or a human) verify without the bytes living in the message. +- Prefer **LINK over SNIPPET** whenever a permalink fully carries the evidence. + +### Size caps (hard ceilings — truncate, never exceed) + +| Field / scope | Soft target | Hard ceiling | On exceed | +|---|---|---|---| +| `SUMMARY` | ≤ 300 chars | 400 chars | Tighten to the finding; drop restatement of the ask | +| `SNIPPET` per ask | ≤ 20 lines | 40 lines | Keep the load-bearing lines; replace the rest with `… (N lines elided — see LINK)` | +| Code diff in a `product_code` snippet | ≤ 1 hunk | 3 hunks | Show changed lines + 3 lines context; link the full PR | +| Whole next-turn `message` | ≤ 200 lines | 400 lines (and ≤ `turnMessageMaxChars`) | Drop `low`-priority asks first; keep every `high` ask's block | +| Asks fulfilled per turn | all `high` + `medium` | — | Defer `low` asks to a later turn rather than truncating a `high` ask | + +Truncation rule of thumb: **never truncate a `high`-priority ask's block to fit a +`low`-priority one.** Drop the low block whole; keep the high block intact. The +whole-message ceiling also honors `turnMessageMaxChars` from +`config/rca.config.json` (the tool caps `message` at 5000 chars). + +### What never goes in a digest + +- Raw log tails, full log output, full file contents, full PR diffs — link or excerpt. +- `test_logs` content of any kind (TFA owns it). +- Credentials, tokens, internal hostnames, or any secret surfaced by an env/secret dump. +- Speculation dressed as a finding. If `FOUND: no`, say what was checked; do not invent a cause. + +--- + +## Unfulfillable asks — report, don't drop + +``` +ASK: <verbatim what> +TYPE: <evidenceType> +FOUND: no +SUMMARY: not-found | unreachable | unavailable | out-of-scope — <one line: what was checked or why blocked> +``` + +- `not-found` — the skill/tool ran but the signal isn't there. State the search performed. +- `unreachable` — the surface was not reachable from this agent context. State which. +- `unavailable` — no capability/skill exists for this `evidenceType` (auto-mode gap result). +- `out-of-scope` — the ask is `test_logs` or otherwise not the agent's to fulfill. + +An all-`unavailable` / all-`not-found` turn still resubmits — TFA decides whether +the gap is fatal (→ BLOCKED) or it can converge anyway (best-effort, lower +confidence). The coordinator does not pre-empt that decision. diff --git a/tests/routing.test.mjs b/tests/routing.test.mjs new file mode 100644 index 0000000..c63b595 --- /dev/null +++ b/tests/routing.test.mjs @@ -0,0 +1,80 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { routeAsk, routeAsks, orderAsks, TEST_LOGS } from "../lib/routing.mjs"; + +const CONFIG = { + evidenceRouting: { + test_logs: { owner: "tfa", skip: true }, + product_code: { capability: "github", discoveryHints: ["github-mcp", "gh"] }, + k8s: { capability: "k8s", discoveryHints: [] }, + other: { capability: "other", discoveryHints: [] }, + }, +}; + +test("test_logs is always skipped (TFA-owned)", () => { + const r = routeAsk({ evidenceType: TEST_LOGS, priority: "high" }, CONFIG, { + github: { available: true }, + }); + assert.equal(r.action, "skip"); + assert.equal(r.reason, "tfa-owned"); +}); + +test("available capability → gather, carrying via", () => { + const r = routeAsk({ evidenceType: "product_code", priority: "high" }, CONFIG, { + github: { available: true, via: "github-mcp" }, + }); + assert.equal(r.action, "gather"); + assert.equal(r.capability, "github"); + assert.equal(r.via, "github-mcp"); +}); + +test("unavailable capability → gap, carrying discovery hints", () => { + const r = routeAsk({ evidenceType: "k8s", priority: "medium" }, CONFIG, { + k8s: { available: false }, + }); + assert.equal(r.action, "gap"); + assert.equal(r.capability, "k8s"); + assert.equal(r.reason, "no-capability"); +}); + +test("capability absent from manifest entirely → gap", () => { + const r = routeAsk({ evidenceType: "k8s", priority: "low" }, CONFIG, {}); + assert.equal(r.action, "gap"); +}); + +test("unknown evidenceType falls back to the 'other' entry", () => { + const r = routeAsk({ evidenceType: "weird", priority: "low" }, CONFIG, { + other: { available: true, via: "best-effort" }, + }); + assert.equal(r.action, "gather"); + assert.equal(r.capability, "other"); +}); + +test("orderAsks sorts high → medium → low, unknown last", () => { + const ordered = orderAsks([ + { what: "c", priority: "low" }, + { what: "a", priority: "high" }, + { what: "d", priority: undefined }, + { what: "b", priority: "medium" }, + ]); + assert.deepEqual( + ordered.map((a) => a.what), + ["a", "b", "c", "d"], + ); +}); + +test("routeAsks buckets a mixed turn in priority order", () => { + const buckets = routeAsks( + [ + { evidenceType: "k8s", priority: "low" }, + { evidenceType: "test_logs", priority: "high" }, + { evidenceType: "product_code", priority: "high" }, + ], + CONFIG, + { github: { available: true, via: "gh" } }, + ); + assert.equal(buckets.skip.length, 1); + assert.equal(buckets.gather.length, 1); + assert.equal(buckets.gap.length, 1); + assert.equal(buckets.gather[0].evidenceType, "product_code"); +}); From cb0d8f6d1408267f06619893c6202891c362b648 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:01:27 +0530 Subject: [PATCH 03/33] feat(rca): pre-flight intake, discovery, CSV/WAL spine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SKILL.md orchestrator spec: mandatory GitHub intake ('I don't have one' → RCA-only; headless missing-input fail-fast), discovery via listTestIds(failed, includeFailureDetail), then cluster/pre-compute/fan-out/report steps. lib/csv-state.mjs is the resumable WAL spine — seed (idempotent, terminal- preserving), claim/heartbeat/flip, reaper, pendingRows — with timestamps injected (workflow-sandbox-safe) and an RFC4180 codec for multiline RCA fields. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- lib/csv-state.mjs | 239 ++++++++++++++++++++++++++++++++++++++ skills/rca-build/SKILL.md | 124 ++++++++++++++++++++ tests/csv-state.test.mjs | 133 +++++++++++++++++++++ 3 files changed, 496 insertions(+) create mode 100644 lib/csv-state.mjs create mode 100644 skills/rca-build/SKILL.md create mode 100644 tests/csv-state.test.mjs diff --git a/lib/csv-state.mjs b/lib/csv-state.mjs new file mode 100644 index 0000000..499f997 --- /dev/null +++ b/lib/csv-state.mjs @@ -0,0 +1,239 @@ +// CSV write-ahead-log spine for the batch (D4 + ideation #7). The CSV is the +// single durable, resumable source of truth for "RCA over ALL failed tests": +// every test is a row, seeded `pending`, claimed by a worker, heartbeated while +// in flight, and flipped to a terminal state with its RCA. A reaper reclaims +// rows stranded by a crashed worker. +// +// Timestamps are passed in as `nowMs` (never read from the clock here) so this +// module is deterministic in tests AND usable from the auto-mode dynamic +// workflow, whose sandbox forbids Date.now(). +// +// In-session / in-workspace only — cross-session durability is deferred. Writes +// are synchronous read-modify-write; Node's single thread serializes them, which +// is sufficient for the in-process 5-concurrent workflow (true multi-process +// locking is out of scope). + +import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs"; +import { dirname } from "node:path"; + +export const COLUMNS = [ + "buildId", + "testRunId", + "testName", + "failure_category", + "error_summary", + "file_path", + "cluster_id", + "rca_done", + "in_flight_worker", + "heartbeat_ts", + "threadId", + "turnId", + "last_evidence_digest", + "root_cause", + "failure_type", + "possible_fix", + "related_prs", + "coverage", + "confidence", + "timestamp", +]; + +export const PENDING = "pending"; +const TERMINAL_STATES = new Set([ + "resolved", + "blocked", + "failed", + "pending-resume", +]); + +// ---- minimal RFC4180-ish CSV codec ---------------------------------------- + +function encodeField(value) { + const s = value == null ? "" : String(value); + if (/[",\r\n]/.test(s)) { + return `"${s.replace(/"/g, '""')}"`; + } + return s; +} + +function encodeRows(rows) { + const lines = [COLUMNS.join(",")]; + for (const row of rows) { + lines.push(COLUMNS.map((c) => encodeField(row[c])).join(",")); + } + return lines.join("\n") + "\n"; +} + +function parseCsv(text) { + const rows = []; + let field = ""; + let record = []; + let inQuotes = false; + for (let i = 0; i < text.length; i++) { + const ch = text[i]; + if (inQuotes) { + if (ch === '"') { + if (text[i + 1] === '"') { + field += '"'; + i++; + } else { + inQuotes = false; + } + } else { + field += ch; + } + } else if (ch === '"') { + inQuotes = true; + } else if (ch === ",") { + record.push(field); + field = ""; + } else if (ch === "\n" || ch === "\r") { + if (ch === "\r" && text[i + 1] === "\n") i++; + record.push(field); + rows.push(record); + field = ""; + record = []; + } else { + field += ch; + } + } + if (field.length > 0 || record.length > 0) { + record.push(field); + rows.push(record); + } + return rows; +} + +// ---- read / write ---------------------------------------------------------- + +export function readRows(csvPath) { + if (!existsSync(csvPath)) return []; + const text = readFileSync(csvPath, "utf8"); + const raw = parseCsv(text).filter((r) => r.some((c) => c.length > 0)); + if (raw.length === 0) return []; + const header = raw[0]; + return raw.slice(1).map((cells) => { + const row = {}; + header.forEach((col, idx) => { + row[col] = cells[idx] ?? ""; + }); + return row; + }); +} + +export function writeRows(csvPath, rows) { + const dir = dirname(csvPath); + if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true }); + writeFileSync(csvPath, encodeRows(rows), "utf8"); +} + +function emptyRow() { + return Object.fromEntries(COLUMNS.map((c) => [c, ""])); +} + +// ---- operations ------------------------------------------------------------- + +// Seed the CSV from a listTestIds(failed, includeFailureDetail) payload. Every +// row starts `pending`. Idempotent: existing rows are preserved (terminal rows +// are never reset; signature columns are refreshed on still-pending rows). New +// tests are appended. Returns the full row set. +export function seed(csvPath, buildId, tests) { + const existing = readRows(csvPath); + const byId = new Map(existing.map((r) => [String(r.testRunId), r])); + + for (const t of tests) { + const id = String(t.test_id ?? t.testRunId); + const sig = t.failure ?? {}; + const prior = byId.get(id); + if (prior) { + // Keep terminal results; only refresh signature on still-pending rows. + if (prior.rca_done === PENDING) { + prior.failure_category = sig.category ?? prior.failure_category; + prior.error_summary = sig.error_summary ?? prior.error_summary; + prior.file_path = sig.file_path ?? prior.file_path; + } + continue; + } + const row = emptyRow(); + row.buildId = buildId; + row.testRunId = id; + row.testName = t.test_name ?? t.testName ?? `Test ${id}`; + row.failure_category = sig.category ?? ""; + row.error_summary = sig.error_summary ?? ""; + row.file_path = sig.file_path ?? ""; + row.rca_done = PENDING; + byId.set(id, row); + existing.push(row); + } + + writeRows(csvPath, existing); + return existing; +} + +// Claim a pending row for `worker`. Refuses (returns false) if another worker +// already owns it. Returns true on success. +export function claim(csvPath, testRunId, worker, nowMs) { + const rows = readRows(csvPath); + const row = rows.find((r) => String(r.testRunId) === String(testRunId)); + if (!row) return false; + if (row.in_flight_worker && row.in_flight_worker !== worker) return false; + if (TERMINAL_STATES.has(row.rca_done)) return false; + row.in_flight_worker = worker; + row.heartbeat_ts = String(nowMs); + writeRows(csvPath, rows); + return true; +} + +export function heartbeat(csvPath, testRunId, worker, nowMs) { + const rows = readRows(csvPath); + const row = rows.find((r) => String(r.testRunId) === String(testRunId)); + if (!row || row.in_flight_worker !== worker) return false; + row.heartbeat_ts = String(nowMs); + writeRows(csvPath, rows); + return true; +} + +// Flip a row to a terminal state, recording the RCA fields and clearing the +// in-flight claim. `fields` carries any of: rca_done, root_cause, failure_type, +// possible_fix, related_prs, threadId, turnId, coverage, confidence, +// last_evidence_digest, cluster_id. +export function flip(csvPath, testRunId, fields, nowMs) { + const rows = readRows(csvPath); + const row = rows.find((r) => String(r.testRunId) === String(testRunId)); + if (!row) return false; + for (const [k, v] of Object.entries(fields)) { + if (COLUMNS.includes(k)) { + row[k] = Array.isArray(v) ? v.join("; ") : (v ?? ""); + } + } + row.in_flight_worker = ""; + row.timestamp = String(nowMs); + writeRows(csvPath, rows); + return true; +} + +// Reclaim rows stranded in flight (heartbeat older than ttlSec) back to pending. +// Returns the testRunIds reclaimed. Run on startup before resuming a batch. +export function reaper(csvPath, ttlSec, nowMs) { + const rows = readRows(csvPath); + const reclaimed = []; + for (const row of rows) { + if (!row.in_flight_worker) continue; + if (TERMINAL_STATES.has(row.rca_done)) continue; + const hb = Number(row.heartbeat_ts); + const stale = !row.heartbeat_ts || nowMs - hb > ttlSec * 1000; + if (stale) { + row.in_flight_worker = ""; + row.rca_done = PENDING; + reclaimed.push(String(row.testRunId)); + } + } + if (reclaimed.length > 0) writeRows(csvPath, rows); + return reclaimed; +} + +// Rows still needing work (pending or reclaimed). The work-list for fan-out. +export function pendingRows(csvPath) { + return readRows(csvPath).filter((r) => r.rca_done === PENDING); +} diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md new file mode 100644 index 0000000..4bf6b34 --- /dev/null +++ b/skills/rca-build/SKILL.md @@ -0,0 +1,124 @@ +--- +name: rca-build +description: Run collaborative root-cause analysis over ALL failed tests of a BrowserStack build. Generic across product and infra. Mandatory pre-flight GitHub intake, then discovery via listTestIds, failure-signature clustering, and per-test RCA via tfaRcaTurn (auto = dynamic workflow / interactive = subagents). Use when a build is red and you want a per-test RCA for every failure in the TRA dashboard. +--- + +# rca-build — batch collaborative RCA over a build + +Drives the `tfaRcaTurn` collaborative loop over **every failed test** of a build +and records a per-test RCA. **TFA owns logs; the client agent owns everything +else** (product code, k8s, kibana, metrics, deploy, ci) — routed by capability, +generic over product and infra. + +This skill is the **build-level orchestrator** (`ai-tfa-orchestrator` role). It +never calls `tfaRcaTurn` itself — it dispatches the `ai-tfa-coordinator` +(test-level) per test/cluster member, which drives the loop and lets TFA author +the dashboard RCA. + +Config (concurrency, turn-cap, paths, evidence registry) lives in +`config/rca.config.json`. State lives in the CSV/WAL spine (`lib/csv-state.mjs`). + +## Step 0 — mode + input + +Parse from `/rca-build` args: the build id and optional `mode=auto|interactive`. + +- No build id present → it is required: + - interactive session → ask the user. + - **headless (`claude -p`) with build id missing → end immediately (fail fast).** +- No mode given → ask the user once (auto vs interactive). In headless, default `auto`. + +## Step 1 — pre-flight intake (F1, mandatory, both modes) + +Ask the user (A1) for, in one pass: + +- product repo name, automation (test) repo name +- working branch, default branch +- the PRs in play (product + automation) +- the build id (if not already supplied) + +Every question is **mandatory to ask** but answerable with **"I don't have one"** +→ record the gap and proceed **RCA-only** (BrowserStack-side evidence + whatever +infra skills exist). Do not block the run on missing GitHub context. + +**Headless rule:** in `claude -p`, any *required* input still missing after +parsing (build id) ends the run immediately. Optional intake answers default to +"none" without prompting. + +## Step 2 — discovery (F2) + +Call the bundled MCP tool: + +``` +listTestIds(buildId=<id>, status="failed", includeFailureDetail=true) +``` + +`includeFailureDetail=true` returns each row's trimmed failure signature +(`failure.{category, error_summary, file_path, …}`) — the seed for clustering, so +no per-test probe turns are needed. + +Seed the CSV/WAL spine from the payload (`lib/csv-state.mjs` → `seed`): one row +per failed test, every row `rca_done=pending`, signature columns populated. +Re-running `seed` on an existing CSV is idempotent and preserves terminal rows +(resume-safe). If `listTestIds` returns empty → write an empty CSV, report "no +failed tests", stop. + +## Step 3 — failure-signature clustering (see references/clustering.md) + +Compute a failure signature per row and assign `cluster_id` (`lib/signature.mjs`). +Each cluster gets one **representative** (full multi-turn loop) and `N−1` +**siblings** (pre-seeded one-turn confirm against their own logs). This collapses +the expensive evidence hunt to O(distinct causes) while every test still lands a +per-test RCA. Singleton clusters are just plain per-test loops. + +## Step 4 — build-evidence pre-compute + capability manifest (see references/evidence-routing.md) + +Once, before fan-out: + +- **Capability manifest** — enumerate the skills/tools the client actually has + into `capability → {available, via}` (GitHub, k8s, logs, metrics, …). Declare + to the user up front what will be **unavailable** ("k8s + metrics not + available"). Every coordinator routes asks against this manifest. +- **Build-level evidence** — compute the last-green→this-build delta (diff, + deploy timeline, suspect-PR window) **once** and pre-seed every coordinator + with the same grounded window. Cache by `(repo, commit-range)`. No "last green" + baseline (never-green suite) → fall back to a configured baseline ref and log it. + +## Step 5 — fan-out (the mode fork) + +Drive the cluster work-list, **`concurrency` (default 5) at a time**: +representatives deep, siblings one-turn-confirm. Eagerly persist to the CSV/WAL +(claim → heartbeat → flip) so the run is resumable. + +- **auto** → run the dynamic workflow `workflows/rca-batch.mjs` (script-orchestrated, + no user input; gap → "unavailable" back to TFA → best-effort finalize). +- **interactive** → spawn `ai-tfa-coordinator` subagents 5 at a time; on an + evidence gap a subagent returns the gap to this orchestrator, which asks the + user (A1), then feeds the answer back. Subagents return compact `RCA_OUTPUT` + blocks, not transcripts (keeps the main context lean for large batches). + +Both modes use the **same** `ai-tfa-coordinator`; only the injected gap-resolver +differs. A coordinator that dies becomes a recorded `failed` row — one stuck test +never sinks the batch (partial-first). + +## Step 6 — report (see references/report-format.md) + +When every row is terminal, render the report (`paths.reportFile`): per-test rows +with status + the **evidence-coverage band** (a RESOLVED built with evidence +unavailable reads as lower confidence than a fully-evidenced one). Degrade, +don't crash — missing fields render as "not available". + +## Resume + +On startup, run the reaper (`lib/csv-state.mjs` → `reaper`) to reclaim rows +stranded `in_flight` by a crashed worker (heartbeat older than +`reaperHeartbeatTtlSec`) back to `pending`, then re-point fan-out at the CSV. +Live `threadId`/`turnId` resume the prior thread; dead threads re-run from +pending. (In-session only — cross-session durability is deferred.) + +## Hard rules + +- Always run the pre-flight intake; never silently skip it (but never block on "I don't have one"). +- Headless + missing required input → end immediately. +- Never call `tfaRcaTurn` from this skill — always via the `ai-tfa-coordinator`. +- Every failed test must end terminal in the CSV — partial-first, no abort-on-one-failure. +- Never gather `test_logs` — TFA owns logs. diff --git a/tests/csv-state.test.mjs b/tests/csv-state.test.mjs new file mode 100644 index 0000000..5a9a60f --- /dev/null +++ b/tests/csv-state.test.mjs @@ -0,0 +1,133 @@ +import { test, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + seed, + readRows, + claim, + heartbeat, + flip, + reaper, + pendingRows, + PENDING, +} from "../lib/csv-state.mjs"; + +let dir; +let csv; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "rca-csv-")); + csv = join(dir, "state.csv"); +}); +afterEach(() => rmSync(dir, { recursive: true, force: true })); + +const TESTS = [ + { + test_id: 101, + test_name: "login", + failure: { category: "Assertion", error_summary: "expected 200", file_path: "a.rb" }, + }, + { test_id: 102, test_name: "checkout", failure: { category: "Timeout" } }, +]; + +test("seed writes one pending row per test with signature columns", () => { + const rows = seed(csv, "build-1", TESTS); + assert.equal(rows.length, 2); + assert.ok(rows.every((r) => r.rca_done === PENDING)); + const login = rows.find((r) => r.testRunId === "101"); + assert.equal(login.failure_category, "Assertion"); + assert.equal(login.error_summary, "expected 200"); + assert.equal(login.buildId, "build-1"); +}); + +test("seed is idempotent — no duplicate rows on re-seed", () => { + seed(csv, "build-1", TESTS); + const rows = seed(csv, "build-1", TESTS); + assert.equal(rows.length, 2); +}); + +test("seed preserves a terminal row on re-seed", () => { + seed(csv, "build-1", TESTS); + flip(csv, 101, { rca_done: "resolved", root_cause: "bad PR" }, 1000); + seed(csv, "build-1", TESTS); + const login = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(login.rca_done, "resolved"); + assert.equal(login.root_cause, "bad PR"); +}); + +test("claim sets the worker; a second worker is refused", () => { + seed(csv, "build-1", TESTS); + assert.equal(claim(csv, 101, "w1", 1000), true); + assert.equal(claim(csv, 101, "w2", 1000), false); + const row = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(row.in_flight_worker, "w1"); +}); + +test("heartbeat updates ts only for the owning worker", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + assert.equal(heartbeat(csv, 101, "w1", 2000), true); + assert.equal(heartbeat(csv, 101, "w2", 3000), false); + assert.equal(readRows(csv).find((r) => r.testRunId === "101").heartbeat_ts, "2000"); +}); + +test("flip records terminal fields, joins related_prs, clears the claim", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + flip( + csv, + 101, + { rca_done: "resolved", root_cause: "PR #7421", related_prs: ["#7421", "#7430"], confidence: "high" }, + 5000, + ); + const row = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(row.rca_done, "resolved"); + assert.equal(row.related_prs, "#7421; #7430"); + assert.equal(row.confidence, "high"); + assert.equal(row.in_flight_worker, ""); + assert.equal(row.timestamp, "5000"); +}); + +test("reaper reclaims only stale in-flight rows", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); // stale + claim(csv, 102, "w2", 9000); // fresh + const ttl = 600; // seconds + const now = 1000 + ttl * 1000 + 1; // just past TTL for w1, fresh for w2 + const reclaimed = reaper(csv, ttl, now); + assert.deepEqual(reclaimed, ["101"]); + const rows = readRows(csv); + assert.equal(rows.find((r) => r.testRunId === "101").in_flight_worker, ""); + assert.equal(rows.find((r) => r.testRunId === "101").rca_done, PENDING); + assert.equal(rows.find((r) => r.testRunId === "102").in_flight_worker, "w2"); +}); + +test("reaper leaves terminal rows alone even if in_flight lingered", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + flip(csv, 101, { rca_done: "resolved" }, 2000); // flip clears in_flight + const reclaimed = reaper(csv, 600, 10_000_000); + assert.deepEqual(reclaimed, []); +}); + +test("pendingRows returns only pending work", () => { + seed(csv, "build-1", TESTS); + flip(csv, 101, { rca_done: "resolved" }, 1000); + const pend = pendingRows(csv); + assert.equal(pend.length, 1); + assert.equal(pend[0].testRunId, "102"); +}); + +test("CSV codec round-trips fields with commas, quotes, newlines", () => { + seed(csv, "build-1", [{ test_id: 200, test_name: "weird" }]); + flip( + csv, + 200, + { rca_done: "resolved", root_cause: 'Failed: "x", got <y>\nsecond line' }, + 1000, + ); + const row = readRows(csv).find((r) => r.testRunId === "200"); + assert.equal(row.root_cause, 'Failed: "x", got <y>\nsecond line'); +}); From bbee37db77095abfda7a1c83a7b42180ef12a901 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:03:37 +0530 Subject: [PATCH 04/33] feat(rca): failure-signature clustering + sibling one-turn-confirm protocol lib/signature.mjs computes signature = normalize(category|error|file) off the U1 discovery payload (folds timestamps/uuids/hex/line:col/numbers), groups rows by signature, picks a deterministic representative (non-flaky, then smallest id), and leaves signal-less rows as their own singletons. references/clustering.md documents the O(causes) protocol: representative runs the full loop; siblings pre-seed a one-turn confirm against their own logs with a fall-back-to-own-loop safeguard (never blindly inherit). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- lib/signature.mjs | 78 ++++++++++++++++++++++ skills/rca-build/references/clustering.md | 60 +++++++++++++++++ tests/signature.test.mjs | 79 +++++++++++++++++++++++ 3 files changed, 217 insertions(+) create mode 100644 lib/signature.mjs create mode 100644 skills/rca-build/references/clustering.md create mode 100644 tests/signature.test.mjs diff --git a/lib/signature.mjs b/lib/signature.mjs new file mode 100644 index 0000000..42dc0ae --- /dev/null +++ b/lib/signature.mjs @@ -0,0 +1,78 @@ +// Failure-signature clustering (ideation #1). A red build's N failures usually +// trace to a handful of causes; clustering collapses the expensive evidence hunt +// to O(distinct causes). The signature is computed from the trimmed failure +// detail U1 surfaces on each listTestIds row (category + first error line + file +// path) — no extra probe turns. +// +// Dependency-free + deterministic (no crypto, no clock, no random) so it is +// usable from the auto-mode workflow sandbox and trivially testable. + +// Normalize a string for signature comparison: lowercase and fold the volatile +// tokens that make two instances of the SAME failure look different (ids, +// timestamps, hex/uuids, line:col, bare numbers). +export function normalize(value) { + return String(value ?? "") + .toLowerCase() + .replace(/\b\d{4}-\d{2}-\d{2}[t ]\d{2}:\d{2}:\d{2}\S*/g, "<ts>") // ISO timestamps + .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g, "<uuid>") + .replace(/0x[0-9a-f]+/g, "<hex>") // memory addresses + .replace(/:\d+(:\d+)?\b/g, ":<line>") // file:line(:col) + .replace(/\d+/g, "<n>") // remaining numbers (incl. unit-suffixed, e.g. 3000ms) + .replace(/\s+/g, " ") + .trim(); +} + +// The signature triple: normalized category | error summary | file path. +export function computeSignature(row) { + const category = normalize(row.failure_category); + const error = normalize(row.error_summary); + const file = normalize(row.file_path); + const sig = `${category}|${error}|${file}`; + return sig.replace(/\|/g, "").trim().length === 0 ? "" : sig; +} + +// Deterministic short id for a signature string (FNV-1a → base36). +function hashId(s) { + let h = 0x811c9dc5; + for (let i = 0; i < s.length; i++) { + h ^= s.charCodeAt(i); + h = Math.imul(h, 0x01000193); + } + return (h >>> 0).toString(36); +} + +// A stable representative for a cluster: prefer a non-flaky member (a flaky test +// is a poor exemplar), then the smallest testRunId. Deterministic. +export function selectRepresentative(members) { + return [...members].sort((a, b) => { + const aFlaky = a.is_flaky === "true" || a.is_flaky === true ? 1 : 0; + const bFlaky = b.is_flaky === "true" || b.is_flaky === true ? 1 : 0; + if (aFlaky !== bFlaky) return aFlaky - bFlaky; + return Number(a.testRunId) - Number(b.testRunId); + })[0]; +} + +// Cluster rows by signature. Mutates each row's `cluster_id`. Rows with no +// signal (empty signature) become their own singleton (never merged into a +// catch-all). Returns { rows, clusters } where each cluster carries its +// representative + siblings. +export function clusterRows(rows) { + const groups = new Map(); + + for (const row of rows) { + const sig = computeSignature(row); + const id = sig === "" ? `solo-${row.testRunId}` : `c-${hashId(sig)}`; + row.cluster_id = id; + if (!groups.has(id)) groups.set(id, { cluster_id: id, signature: sig, members: [] }); + groups.get(id).members.push(row); + } + + const clusters = []; + for (const group of groups.values()) { + const representative = selectRepresentative(group.members); + const siblings = group.members.filter((m) => m !== representative); + clusters.push({ ...group, representative, siblings }); + } + + return { rows, clusters }; +} diff --git a/skills/rca-build/references/clustering.md b/skills/rca-build/references/clustering.md new file mode 100644 index 0000000..66face8 --- /dev/null +++ b/skills/rca-build/references/clustering.md @@ -0,0 +1,60 @@ +# Failure-signature clustering + +Why: a red build's N failures usually trace to a handful of causes (one bad +PR/deploy/shared helper). Running the full collaborative loop once per *cause* +instead of once per *test* turns the dominant cost from **O(tests) → O(distinct +causes)** — the only thing that makes "RCA for ALL failed tests, even thousands" +feasible. But **every failed test must still show a per-test RCA in the TRA +dashboard**, so clustering collapses the *evidence hunt*, not the *output*. + +The logic lives in `lib/signature.mjs`; this file is the protocol. + +## The signature + +Computed from the trimmed failure detail `listTestIds(includeFailureDetail=true)` +already returns on each row — **no extra probe turns**: + +``` +signature = normalize(failure_category) | normalize(error_summary) | normalize(file_path) +``` + +`normalize` folds the volatile tokens that make two instances of the *same* +failure look different: ISO timestamps, UUIDs, hex/memory addresses, `file:line:col`, +and bare numbers. So `timeout after 3000ms on node-7` and `timeout after 5000ms +on node-2` share a signature. + +A row with **no signal** (empty category, error, and path) is **not** merged into +a catch-all — it becomes its own singleton (`solo-<testRunId>`). Better an +un-clustered test than a wrong cluster. + +## Representative + siblings + +Each cluster gets: + +- **Representative** — a stable exemplar (non-flaky preferred, then smallest + `testRunId`). Runs the **full multi-turn `ai-tfa-coordinator` loop** → + confirmed root cause + culprit `related_prs`. +- **Siblings** (`N−1`) — each runs its **own** coordinator, **pre-seeded** with + the representative's `root_cause` + suspect PRs. TFA confirms the hypothesis + **against that sibling's own logs in a single turn** → a logs-grounded per-test + RCA in the dashboard at minimal cost. + +Net cost per cluster: **1 deep investigation + (N−1) one-turn confirms.** + +## The safeguard — never blindly inherit + +Distinct failures can share an error string. A sibling's pre-seed turn is a +*hypothesis to confirm*, not a verdict to copy: + +- TFA `RESOLVED`s the sibling in one turn → logs-grounded inheritance, cheap. +- TFA returns `NEEDS_INFO` / `BLOCKED` (the hypothesis does not hold for this + test's logs) → the sibling **falls back to its own full loop**. The + representative's cause is never stamped onto a sibling without log confirmation. + +This keeps correctness independent of the cost optimization: worst case, every +sibling runs its own full loop (same as no clustering); best case, one deep run +covers the whole cluster. + +## Singletons + +A cluster of one is just a plain per-test loop — no pre-seed, no confirm step. diff --git a/tests/signature.test.mjs b/tests/signature.test.mjs new file mode 100644 index 0000000..f721167 --- /dev/null +++ b/tests/signature.test.mjs @@ -0,0 +1,79 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + normalize, + computeSignature, + selectRepresentative, + clusterRows, +} from "../lib/signature.mjs"; + +function row(id, extra = {}) { + return { + testRunId: String(id), + failure_category: "Assertion", + error_summary: "expected 200 but got 500", + file_path: "spec/login.rb", + is_flaky: "false", + ...extra, + }; +} + +test("normalize folds timestamps, uuids, hex, line:col, and numbers", () => { + assert.equal(normalize("Error at line :42:7"), "error at line :<line>"); + assert.equal(normalize("got 500 at 0xAF3"), "got <n> at <hex>"); + assert.equal( + normalize("failed 2026-06-23T10:00:00Z"), + "failed <ts>", + ); +}); + +test("identical category+error+path → same cluster", () => { + const { clusters } = clusterRows([row(1), row(2)]); + assert.equal(clusters.length, 1); + assert.equal(clusters[0].members.length, 2); +}); + +test("numbers in the error are folded so siblings still cluster", () => { + const a = row(1, { error_summary: "timeout after 3000ms on node-7" }); + const b = row(2, { error_summary: "timeout after 5000ms on node-2" }); + assert.equal(computeSignature(a), computeSignature(b)); + const { clusters } = clusterRows([a, b]); + assert.equal(clusters.length, 1); +}); + +test("distinct failures → distinct clusters", () => { + const a = row(1, { error_summary: "null pointer in Foo" }); + const b = row(2, { error_summary: "connection refused" }); + const { clusters } = clusterRows([a, b]); + assert.equal(clusters.length, 2); +}); + +test("rows with no signal become their own singletons (no catch-all merge)", () => { + const a = { testRunId: "1", failure_category: "", error_summary: "", file_path: "" }; + const b = { testRunId: "2", failure_category: "", error_summary: "", file_path: "" }; + const { clusters } = clusterRows([a, b]); + assert.equal(clusters.length, 2); + assert.ok(clusters.every((c) => c.cluster_id.startsWith("solo-"))); +}); + +test("singleton cluster has a representative and no siblings", () => { + const { clusters } = clusterRows([row(1)]); + assert.equal(clusters[0].siblings.length, 0); + assert.equal(clusters[0].representative.testRunId, "1"); +}); + +test("representative is deterministic: non-flaky, then smallest testRunId", () => { + const members = [ + row(5, { is_flaky: "true" }), + row(9, { is_flaky: "false" }), + row(7, { is_flaky: "false" }), + ]; + assert.equal(selectRepresentative(members).testRunId, "7"); +}); + +test("clusterRows stamps cluster_id onto every row", () => { + const rows = [row(1), row(2, { error_summary: "different" })]; + clusterRows(rows); + assert.ok(rows.every((r) => r.cluster_id)); + assert.notEqual(rows[0].cluster_id, rows[1].cluster_id); +}); From 7ddb2c222bd9b3b479a3bce82d3ed0c5e75de69c Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:05:29 +0530 Subject: [PATCH 05/33] feat(rca): build-evidence pre-compute + cache + capability manifest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buildManifest enumerates the client's discovered capabilities once into capability→{available,via}, declared to the user + TFA so no evidence is asked for that the client provably can't get. lib/evidence-cache.mjs computes the last-green→this-build delta once and caches by (repo,range,evidenceType) — fresh per-run Map, no module globals (multi-tenant-safe) — with resolveBaseline for the never-green fallback. Routes the same grounded window into every coordinator. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- lib/evidence-cache.mjs | 47 ++++++++++++ lib/routing.mjs | 31 ++++++++ .../rca-build/references/evidence-routing.md | 29 +++++++ tests/evidence.test.mjs | 76 +++++++++++++++++++ 4 files changed, 183 insertions(+) create mode 100644 lib/evidence-cache.mjs create mode 100644 tests/evidence.test.mjs diff --git a/lib/evidence-cache.mjs b/lib/evidence-cache.mjs new file mode 100644 index 0000000..b9c9523 --- /dev/null +++ b/lib/evidence-cache.mjs @@ -0,0 +1,47 @@ +// Build-level evidence cache (ideation #2). "Diff since last green", "deploy +// timeline", "PRs in the suspect window" are properties of the BUILD, not the +// test — yet a naive loop re-fetches them per test. Compute once, cache by +// (repo, commit-range, evidenceType), and pre-seed every coordinator with the +// same grounded suspect window. Collapses N×M redundant git/infra calls to ~M. +// +// The cache is created fresh per run (function-scoped Map — never a module-level +// global), so it holds no cross-run/cross-user state: in-workspace, single +// session, multi-tenant-safe by construction. + +export function makeEvidenceCache() { + const store = new Map(); + const keyOf = (repo, range, evidenceType) => + `${repo ?? ""}@@${range ?? ""}@@${evidenceType ?? ""}`; + + return { + has(repo, range, evidenceType) { + return store.has(keyOf(repo, range, evidenceType)); + }, + get(repo, range, evidenceType) { + return store.get(keyOf(repo, range, evidenceType)); + }, + set(repo, range, evidenceType, value) { + store.set(keyOf(repo, range, evidenceType), value); + return value; + }, + // Compute-once: run `fn` only on a cache miss; reuse on every later call. + async compute(repo, range, evidenceType, fn) { + const k = keyOf(repo, range, evidenceType); + if (store.has(k)) return store.get(k); + const value = await fn(); + store.set(k, value); + return value; + }, + size() { + return store.size; + }, + }; +} + +// Resolve the baseline ref for the last-green→this-build delta. When there is no +// "last green" (e.g. a never-green flaky suite) fall back to a configured ref and +// flag it so the report can note the weaker grounding. +export function resolveBaseline(lastGreenRef, fallbackRef) { + if (lastGreenRef) return { ref: lastGreenRef, isFallback: false }; + return { ref: fallbackRef ?? null, isFallback: true }; +} diff --git a/lib/routing.mjs b/lib/routing.mjs index 291738e..ec7c4e7 100644 --- a/lib/routing.mjs +++ b/lib/routing.mjs @@ -73,3 +73,34 @@ export function routeAsks(asks, config, manifest = {}) { } return buckets; } + +// ---- capability manifest (ideation #3) ------------------------------------- + +// Build the capability manifest ONCE per run from the capabilities the client +// agent actually discovered. `discovered` is a list of +// { capability, via } the orchestrator collected by asking "what skills/tools +// are available?". Every capability the routing registry references (except the +// TFA-owned test_logs) appears in the manifest, marked available iff discovered. +// Declaring this to TFA lets it avoid asking for evidence the client can't get. +export function buildManifest(config, discovered = []) { + const byCap = new Map(discovered.map((d) => [d.capability, d])); + const manifest = {}; + for (const entry of Object.values(config?.evidenceRouting ?? {})) { + if (entry.skip || entry.owner === "tfa") continue; + const cap = entry.capability; + if (!cap || cap in manifest) continue; + const found = byCap.get(cap); + manifest[cap] = found + ? { available: true, via: found.via ?? null } + : { available: false, via: null }; + } + return manifest; +} + +// Capabilities that will be unavailable this run — declared to the user up front +// ("k8s + metrics not available") and to TFA so it plans asks around them. +export function unavailableCapabilities(manifest) { + return Object.entries(manifest) + .filter(([, v]) => !v.available) + .map(([cap]) => cap); +} diff --git a/skills/rca-build/references/evidence-routing.md b/skills/rca-build/references/evidence-routing.md index e6cc4d0..87fb255 100644 --- a/skills/rca-build/references/evidence-routing.md +++ b/skills/rca-build/references/evidence-routing.md @@ -131,3 +131,32 @@ SUMMARY: not-found | unreachable | unavailable | out-of-scope — <one line: wha An all-`unavailable` / all-`not-found` turn still resubmits — TFA decides whether the gap is fatal (→ BLOCKED) or it can converge anyway (best-effort, lower confidence). The coordinator does not pre-empt that decision. + +--- + +## Capability manifest (built once per run) + +Rather than re-discover "is there a kibana skill?" on every ask across every +test, the orchestrator enumerates the client's available skills/tools **once** up +front into a manifest (`lib/routing.mjs` → `buildManifest`): + +``` +{ github: {available: true, via: "github-mcp"}, k8s: {available: false}, ... } +``` + +- Every ask routes against this manifest — reproducible, no per-ask discovery. +- The orchestrator **declares the unavailable capabilities to the user** up front + ("k8s + metrics will be unavailable") and includes them in the first turn so + TFA plans asks around what's obtainable. +- Frozen at run start. A skill appearing mid-run is not picked up until the next run. + +## Build-level evidence cache (compute once) + +"Diff since last green", "deploy timeline", and "PRs in the suspect window" are +properties of the **build**, not the test. The orchestrator computes the +last-green→this-build delta **once** (`lib/evidence-cache.mjs`), caches it by +`(repo, commit-range, evidenceType)`, and pre-seeds every coordinator with the +same grounded suspect window — collapsing N×M redundant git/infra calls to ~M and +front-loading the highest-signal evidence so many tests RESOLVE before any infra +ask fires. No "last green" (never-green suite) → fall back to a configured +baseline ref and note the weaker grounding in the report. diff --git a/tests/evidence.test.mjs b/tests/evidence.test.mjs new file mode 100644 index 0000000..e01b06a --- /dev/null +++ b/tests/evidence.test.mjs @@ -0,0 +1,76 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { buildManifest, unavailableCapabilities } from "../lib/routing.mjs"; +import { makeEvidenceCache, resolveBaseline } from "../lib/evidence-cache.mjs"; + +const CONFIG = { + evidenceRouting: { + test_logs: { owner: "tfa", skip: true }, + product_code: { capability: "github" }, + deploy: { capability: "github" }, + k8s: { capability: "k8s" }, + metrics: { capability: "metrics" }, + other: { capability: "other" }, + }, +}; + +test("buildManifest marks discovered capabilities available with via", () => { + const manifest = buildManifest(CONFIG, [ + { capability: "github", via: "github-mcp" }, + ]); + assert.equal(manifest.github.available, true); + assert.equal(manifest.github.via, "github-mcp"); + assert.equal(manifest.k8s.available, false); +}); + +test("buildManifest excludes the TFA-owned test_logs capability", () => { + const manifest = buildManifest(CONFIG, []); + assert.ok(!("undefined" in manifest)); + assert.ok(!Object.keys(manifest).includes("test_logs")); +}); + +test("buildManifest dedupes capabilities shared by multiple evidence types", () => { + // product_code + deploy both map to github → one manifest entry + const manifest = buildManifest(CONFIG, [{ capability: "github" }]); + assert.equal(Object.keys(manifest).filter((k) => k === "github").length, 1); +}); + +test("unavailableCapabilities lists what the client can't get", () => { + const manifest = buildManifest(CONFIG, [{ capability: "github" }]); + const unavailable = unavailableCapabilities(manifest).sort(); + assert.deepEqual(unavailable, ["k8s", "metrics", "other"]); +}); + +test("evidence cache computes once and reuses across calls", async () => { + const cache = makeEvidenceCache(); + let calls = 0; + const fn = async () => { + calls++; + return { prs: ["#1"] }; + }; + const a = await cache.compute("repo", "abc..def", "deploy", fn); + const b = await cache.compute("repo", "abc..def", "deploy", fn); + assert.equal(calls, 1); + assert.deepEqual(a, b); + assert.equal(cache.size(), 1); +}); + +test("evidence cache key distinguishes commit ranges", async () => { + const cache = makeEvidenceCache(); + let calls = 0; + const fn = async () => ++calls; + await cache.compute("repo", "r1", "deploy", fn); + await cache.compute("repo", "r2", "deploy", fn); + assert.equal(calls, 2); +}); + +test("resolveBaseline uses last-green when present, else flags fallback", () => { + assert.deepEqual(resolveBaseline("v1.2.3", "main"), { + ref: "v1.2.3", + isFallback: false, + }); + assert.deepEqual(resolveBaseline(null, "main"), { + ref: "main", + isFallback: true, + }); +}); From d6f0452298a8a923105bf473388b7d5bfd1b50ce Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:07:26 +0530 Subject: [PATCH 06/33] feat(rca): auto-mode dynamic workflow (rca-batch) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit workflows/rca-batch.mjs orchestrates the batch in auto mode: a pipeline over clusters dispatches ai-tfa-coordinator agents — representative full loop → siblings one-turn-confirm, no barrier between stages — with a structured RCA schema. Sandbox-correct: does no state I/O itself (orchestrator passes the clustered work-list + manifest + pre-computed build evidence via args; each coordinator agent persists its own CSV row eagerly). Gap → 'unavailable' back to TFA, no user prompt. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- workflows/rca-batch.mjs | 130 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 workflows/rca-batch.mjs diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs new file mode 100644 index 0000000..e577dbe --- /dev/null +++ b/workflows/rca-batch.mjs @@ -0,0 +1,130 @@ +export const meta = { + name: "rca-batch", + description: + "Drive collaborative RCA over all failed tests of a build (auto mode): cluster representatives run the full loop, siblings one-turn-confirm, ~5 concurrent.", + phases: [ + { title: "Representatives", detail: "full multi-turn RCA per cluster" }, + { title: "Siblings", detail: "one-turn confirm against own logs" }, + ], +}; + +// AUTO MODE orchestration (D2). This is a dynamic-workflow script: it runs in the +// Workflow sandbox (no filesystem, no Date.now/Math.random, agent()/pipeline() +// as globals). It therefore does NO state I/O itself — the orchestrator seeds the +// CSV, clusters, and builds the manifest in normal context and passes the +// work-list via `args`; each dispatched `ai-tfa-coordinator` agent (which HAS +// tool access) claims + flips its own CSV row eagerly (WAL); this script +// orchestrates concurrency and returns the structured results for reconciliation. +// +// args shape: +// { +// csvPath, buildId, mode: "auto", +// manifest: { capability: { available, via } }, +// buildEvidence: { baselineRef, suspectWindow, ... }, // pre-computed once +// clusters: [ +// { cluster_id, representative: { testRunId, testName, error_summary }, +// siblings: [ { testRunId, testName, error_summary } ] } +// ] +// } + +const RCA_SCHEMA = { + type: "object", + required: ["testRunId", "status"], + properties: { + testRunId: { type: "string" }, + status: { enum: ["RESOLVED", "BLOCKED", "PENDING", "failed"] }, + confidence: { enum: ["high", "medium", "low", "unknown"] }, + root_cause: { type: "string" }, + possible_fix: { type: "string" }, + related_prs: { type: "array", items: { type: "string" } }, + suspect_signals: { type: "array", items: { type: "string" } }, + threadId: { type: "string" }, + turnId: { type: "string" }, + turns_used: { type: "number" }, + asks_fulfilled: { type: "array", items: { type: "string" } }, + asks_skipped: { type: "array", items: { type: "string" } }, + asks_unavailable: { type: "array", items: { type: "string" } }, + cluster_id: { type: "string" }, + }, + additionalProperties: true, +}; + +const ctx = args ?? {}; +const clusters = ctx.clusters ?? []; +const shared = [ + `CSV state file: ${ctx.csvPath}`, + `Capability manifest: ${JSON.stringify(ctx.manifest ?? {})}`, + `Build-level evidence (pre-computed once, reuse — do not re-fetch): ${JSON.stringify(ctx.buildEvidence ?? {})}`, + `Mode: auto — on an evidence gap with no capability, report "unavailable" back to TFA (NEVER prompt a user). Best-effort finalize.`, + `Persist eagerly to the CSV: claim your row before turn 1, flip it on terminal (lib/csv-state.mjs).`, +].join("\n"); + +function repPrompt(cluster) { + const r = cluster.representative; + return [ + `You are the ai-tfa-coordinator for cluster ${cluster.cluster_id}.`, + `Run the FULL collaborative RCA loop for the representative test.`, + `testRunId=${r.testRunId} testName=${r.testName ?? ""}`, + `error_digest: ${r.error_summary ?? "(none)"}`, + shared, + `Return the structured RCA_OUTPUT for this test.`, + ].join("\n"); +} + +function siblingPrompt(sibling, repResult, cluster) { + return [ + `You are the ai-tfa-coordinator for a SIBLING of cluster ${cluster.cluster_id}.`, + `Pre-seed: the representative resolved as:`, + ` root_cause: ${repResult?.root_cause ?? "(representative did not resolve)"}`, + ` related_prs: ${JSON.stringify(repResult?.related_prs ?? [])}`, + `State this hypothesis on turn 1 and ask TFA to CONFIRM it against THIS test's own logs.`, + `If TFA confirms in one turn → done. If it does NOT (NEEDS_INFO/BLOCKED), fall back to the full loop — never blindly inherit.`, + `testRunId=${sibling.testRunId} testName=${sibling.testName ?? ""}`, + `error_digest: ${sibling.error_summary ?? "(none)"}`, + shared, + `Return the structured RCA_OUTPUT for this test.`, + ].join("\n"); +} + +log(`Auto-mode batch: ${clusters.length} cluster(s) over build ${ctx.buildId ?? "?"}`); + +// Pipeline: each cluster flows representative → siblings independently (no barrier +// between stages), so a small cluster's siblings confirm while a big cluster's +// representative is still looping. Concurrency is bounded by the workflow runtime +// (~min(16, cores-2)); config.concurrency (5) is the intended soft target. +const results = await pipeline( + clusters, + (cluster) => + agent(repPrompt(cluster), { + label: `rep:${cluster.representative.testRunId}`, + phase: "Representatives", + agentType: "ai-tfa-coordinator", + schema: RCA_SCHEMA, + }).then((rca) => ({ cluster, rca })), + ({ cluster, rca }) => + parallel( + (cluster.siblings ?? []).map((sib) => () => + agent(siblingPrompt(sib, rca, cluster), { + label: `sib:${sib.testRunId}`, + phase: "Siblings", + agentType: "ai-tfa-coordinator", + schema: RCA_SCHEMA, + }), + ), + ).then((sibs) => ({ + cluster_id: cluster.cluster_id, + representative: rca, + siblings: sibs.filter(Boolean), + })), +); + +const flat = results.filter(Boolean); +const all = flat.flatMap((r) => [r.representative, ...(r.siblings ?? [])]).filter(Boolean); +const byStatus = all.reduce((acc, r) => { + acc[r.status] = (acc[r.status] ?? 0) + 1; + return acc; +}, {}); + +log(`Auto-mode batch complete: ${all.length} test(s) — ${JSON.stringify(byStatus)}`); + +return { clusters: flat.length, tests: all.length, byStatus, results: flat }; From 28ebc1d29c2d6501098ea446e105c5f01e97fd18 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:08:45 +0530 Subject: [PATCH 07/33] =?UTF-8?q?feat(rca):=20interactive=20mode=20?= =?UTF-8?q?=E2=80=94=20subagents=20with=20user-in-the-loop=20gap-return?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit references/interactive-mode.md specifies the orchestrator loop: spawn ai-tfa-coordinator subagents 5 at a time; a subagent cannot pause to prompt the user, so on an evidence gap it ends early with a GAP_OUTPUT carrying resume handles (threadId+turnId); the orchestrator asks A1, then re-dispatches with resume= and the answer. Same coordinator as auto — only the gap action differs. Compact blocks not transcripts (lean main context); partial-first; auto-first/ escalate-the-residue noted. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- agents/ai-tfa-coordinator.md | 31 ++++++++-- skills/rca-build/SKILL.md | 7 ++- .../rca-build/references/interactive-mode.md | 56 +++++++++++++++++++ 3 files changed, 86 insertions(+), 8 deletions(-) create mode 100644 skills/rca-build/references/interactive-mode.md diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index e2045fb..4d42f17 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -71,12 +71,33 @@ capability available for that `evidenceType`): - **auto** → emit an `unavailable` block back to TFA (no user prompt). TFA finalizes best-effort with lower confidence. -- **interactive** → **return the gap to the caller** (the main agent), which asks - the user (A1) for that data, then feeds the answer back. A subagent cannot - prompt the user itself. +- **interactive** → a subagent cannot pause to prompt the user, so **end the run + early and return a `GAP_OUTPUT` block** (status `PENDING`) carrying the resume + handles + the gap. The orchestrator asks A1, then **re-dispatches a coordinator + with `resume={threadId, turnId}`** and the answer digested into the next turn. + See `references/interactive-mode.md`. -Everything else — the loop, routing, digest, caps, output — is identical across -modes. Do not fork the loop; only the gap action differs. +`GAP_OUTPUT` block (interactive gap only): + +``` +GAP_OUTPUT_START +## testRunId +<integer> +## thread_id +<threadId> +## turn_id +<turnId> # resume handle +## gap +- evidenceType: <type> +- what: <verbatim ask `what`> +- why: <verbatim ask `why`> +GAP_OUTPUT_END +``` + +Everything else — the loop, routing, digest, caps, terminal output — is identical +across modes. Do not fork the loop; only the gap action differs. When all gaps in +a turn are resolvable (gathered or user-answered), the loop proceeds normally to a +terminal `RCA_OUTPUT`. ## The loop diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index 4bf6b34..5248bee 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -92,9 +92,10 @@ representatives deep, siblings one-turn-confirm. Eagerly persist to the CSV/WAL - **auto** → run the dynamic workflow `workflows/rca-batch.mjs` (script-orchestrated, no user input; gap → "unavailable" back to TFA → best-effort finalize). - **interactive** → spawn `ai-tfa-coordinator` subagents 5 at a time; on an - evidence gap a subagent returns the gap to this orchestrator, which asks the - user (A1), then feeds the answer back. Subagents return compact `RCA_OUTPUT` - blocks, not transcripts (keeps the main context lean for large batches). + evidence gap a subagent ends early with a `GAP_OUTPUT` (resume handles), and + this orchestrator asks the user (A1) then re-dispatches with `resume=`. Subagents + return compact blocks, not transcripts (keeps the main context lean for large + batches). Full protocol: `references/interactive-mode.md`. Both modes use the **same** `ai-tfa-coordinator`; only the injected gap-resolver differs. A coordinator that dies becomes a recorded `failed` row — one stuck test diff --git a/skills/rca-build/references/interactive-mode.md b/skills/rca-build/references/interactive-mode.md new file mode 100644 index 0000000..493ea65 --- /dev/null +++ b/skills/rca-build/references/interactive-mode.md @@ -0,0 +1,56 @@ +# Interactive mode — subagents with a user in the loop + +Interactive mode (D2) puts the human (A1) in the loop **only at the orchestrator +layer**. The main session spawns `ai-tfa-coordinator` subagents to investigate in +parallel; when a subagent needs evidence it can't get, it hands the gap back up to +the orchestrator, which asks the user and feeds the answer down. + +This is the **same coordinator** the auto workflow uses — only the gap-resolver +differs (auto → "unavailable"; interactive → return the gap). + +## Why a subagent can't just "ask the user" + +A dispatched subagent runs to completion and returns one final message — it +cannot pause mid-run, prompt the user, and resume. So the gap-return is modeled +as **early termination with resume handles**, and the orchestrator drives the +ask-and-resume loop. + +## The orchestrator loop (per batch of ≤ `concurrency`, default 5) + +``` +1. Take the next ≤5 pending work items (representatives first, then siblings). +2. Dispatch one ai-tfa-coordinator subagent per item, mode=interactive, passing + the manifest + pre-computed build evidence + (for siblings) the pre-seed. +3. Each subagent runs its loop until either: + - a terminal status → returns RCA_OUTPUT (the orchestrator flips the CSV row), or + - an interactive GAP → returns GAP_OUTPUT (status=PENDING) carrying: + { testRunId, threadId, turnId, gap: { evidenceType, what, why } } +4. For each GAP_OUTPUT: ASK A1 for that evidence (one focused question). + - A1 answers → re-dispatch a coordinator with resume={threadId,turnId} and + the answer digested into the next turn's message. Continue its loop. + - A1 has nothing → tell the coordinator to report "unavailable" on resume + (degrade exactly like auto for that one ask). +5. Repeat until every row is terminal. Then dispatch the next batch. +``` + +## Aggregation discipline (large batches) + +Subagents return **compact `RCA_OUTPUT` / `GAP_OUTPUT` blocks, never transcripts** +— mirroring the auto workflow's "results in script vars" rule — so the main +agent's context stays lean even over hundreds of tests. The orchestrator never +holds full per-test loop transcripts; it holds one block per test. + +## Partial-first + +A subagent that dies becomes a recorded `failed` row (the orchestrator +synthesizes it). One stuck test never sinks the batch — same contract as auto. + +## When to prefer interactive over auto + +- The client is missing infra skills the failures clearly need (k8s/kibana), and + the user can supply that evidence by hand. +- The user wants to steer or sign off mid-run. + +Otherwise auto is cheaper (no human round-trips). Both write the same CSV rows +and the same report, so a run can start auto and the residual BLOCKED/gap tests +can be re-run interactively (the auto-first / escalate-the-residue pattern). From e8b70e644225bc448fc47a0508b61223456cf07a Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:09:44 +0530 Subject: [PATCH 08/33] feat(rca): suspect-PR falsification packet + GitHub-evidence spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit references/github-evidence.md specifies exactly what each github ask needs (diff-since-baseline, PRs-in-window touching the failing path, blame, deploy timing) and the discovery order GitHub MCP → gh → degrade — no shipped forensics harness. Adds the adversarial falsification protocol (path overlap / deploy-state guard / direction) so only verdict:supported suspects enter related_prs; ruled-out suspects stay as disconfirming evidence. Coordinator runs it for product_code/ deploy/ci asks, reusing the pre-computed build evidence. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- agents/ai-tfa-coordinator.md | 12 +++ .../rca-build/references/github-evidence.md | 77 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 skills/rca-build/references/github-evidence.md diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 4d42f17..60b9753 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -99,6 +99,18 @@ across modes. Do not fork the loop; only the gap action differs. When all gaps i a turn are resolvable (gathered or user-answered), the loop proceeds normally to a terminal `RCA_OUTPUT`. +## Suspect-PR falsification (github asks) + +For `product_code` / `deploy` / `ci` asks, follow `references/github-evidence.md`: +gather the **exact** evidence (diff-since-baseline, PRs-in-window touching the +failing path, blame, deploy timing) via **GitHub MCP → `gh` → degrade**, and for +each candidate suspect **try to disprove it** (path overlap? shipped before the +failure window? behind an OFF flag?). Feed both supporting *and* disconfirming +evidence back as a structured suspect packet; only `verdict: supported` suspects +belong in `related_prs`. Reuse the pre-computed build-level evidence — do not +re-fetch per test. Never fabricate a PR when the github capability is unavailable +— emit an `unavailable` block. + ## The loop ``` diff --git a/skills/rca-build/references/github-evidence.md b/skills/rca-build/references/github-evidence.md new file mode 100644 index 0000000..fa24aaa --- /dev/null +++ b/skills/rca-build/references/github-evidence.md @@ -0,0 +1,77 @@ +# GitHub evidence — what to gather, and how to rule a suspect OUT + +The worst automated-RCA outcome is **confidently blaming an innocent PR**. This +file is the contract for `product_code` / `deploy` / `ci` asks (the `github` +capability): the **exact** evidence to gather, and a **falsification protocol** +that tries to *disprove* each suspect before it enters `related_prs`. + +> We do **not** ship a GitHub forensics harness or MCP tool. We specify what's +> needed and use whatever the client already has — **GitHub MCP if available, +> else `gh`, else degrade** to an `unavailable` block. + +## Capability discovery (in order) + +1. **GitHub MCP** (`mcp__github__*`) — preferred for structured PR/diff/blame queries. +2. **`gh` CLI** — fall back for git-graph operations (`gh pr list --search`, + `gh api`, `merge-base`, ancestry) and anything the MCP doesn't cover. +3. **Neither** → emit an `unavailable` block for the ask (do not fabricate a PR). + +The orchestrator records which is present in the capability manifest +(`capability: github → { available, via }`); route every github ask against it. + +## Evidence each ask needs (be specific — no fishing) + +| Ask intent | Gather exactly | +|---|---| +| "Did `<X>` change since the last passing run?" | the diff of `<X>`'s file/function between the **baseline ref** (last-green, or the configured fallback) and the build's commit — not the whole repo diff | +| "Which PRs are suspect?" | PRs **merged in the window** `(baselineRef, build commit]` that **touch the failing code path** — intersect changed files with the failing file/function | +| "Who/what last changed the failing line?" | `blame` on the specific failing lines (from the test's `file_path` + the error) | +| "What shipped to the run's env before the failure?" | deploy timeline (`gh` releases/tags + the env's deploy record); compare deploy time vs. the run's `started_at` | +| "Did CI change?" | the workflow-file diff + recent `gh run` history for the failing job | + +Scope everything by the failing test's `file_path` + the error summary. The +build-level evidence (diff-since-last-green, PR window) is **pre-computed once** +and passed in — reuse it; do not re-fetch per test. + +## Falsification protocol — rule out, don't just rule in + +For **each** candidate suspect PR, try to **break** the hypothesis: + +1. **Path overlap.** Do the PR's changed hunks actually touch the failing code + path (the function/line in the stack)? No overlap → **ruled out**. +2. **Deployment-state guard.** Was the PR's code actually **live** in the run's + env at `started_at`? If it shipped *after* the failure window, or sits behind + an **OFF** flag, it could not have caused this failure → **ruled out**. +3. **Direction.** Does the change plausibly produce *this* error (e.g. a validator + tightened to reject the input the test sends)? If the change is unrelated to + the symptom → **weak**, mark accordingly. + +Feed **both supporting and disconfirming** evidence back to TFA. A suspect that +survives 1–3 is a real candidate; one that fails any is reported as ruled-out +(with the reason), **not** dropped silently. + +## The suspect packet (structured, not free text) + +Each surviving/ruled-out suspect is one structured block so `related_prs` +populates deterministically: + +``` +SUSPECT: + pr: <#number> + files: <changed files overlapping the failing path> + hunks: <the 1-3 load-bearing changed hunks — see digest size caps> + author: <login> + merged_at: <ts> vs last_green: <ts> vs started_at: <ts> + verdict: supported | ruled-out (<reason: no-path-overlap | shipped-after | behind-off-flag | unrelated>) + link: <PR permalink> +``` + +Only `verdict: supported` suspects should end up in TFA's `related_prs`. Ruled-out +suspects stay in the thread as disconfirming evidence so TFA (and a human) can see +the elimination, not just the conclusion. + +## Digest discipline + +Same caps as `references/evidence-routing.md`: prefer a PR **link** over pasting a +diff; at most 1 hunk (3 hard) per `product_code` snippet; never paste a full diff. +The packet is *findings*, not the haystack. From c5a4e9ecb219a3fd52ca7bdecc0365069f543b6d Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:11:34 +0530 Subject: [PATCH 09/33] feat(rca): coverage stamp + degrade-don't-crash report (resume reaper in U4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lib/coverage.mjs derives a per-row evidence-coverage band — TFA confidence capped by coverage (full keeps it, partial→medium, thin→low) so a RESOLVED built with evidence unavailable reads as lower confidence BECAUSE of the gap. lib/report.mjs renders the CSV to markdown: status counts + per-test table + coverage caveats, degrading missing fields to 'not available' and never crashing on an empty/partial batch. report-format.md documents the stamp, layout, and the startup reaper resume path. Blast-radius digest explicitly deferred. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- lib/coverage.mjs | 39 +++++++ lib/report.mjs | 69 ++++++++++++ skills/rca-build/references/report-format.md | 45 ++++++++ tests/coverage-report.test.mjs | 108 +++++++++++++++++++ 4 files changed, 261 insertions(+) create mode 100644 lib/coverage.mjs create mode 100644 lib/report.mjs create mode 100644 skills/rca-build/references/report-format.md create mode 100644 tests/coverage-report.test.mjs diff --git a/lib/coverage.mjs b/lib/coverage.mjs new file mode 100644 index 0000000..caff2d2 --- /dev/null +++ b/lib/coverage.mjs @@ -0,0 +1,39 @@ +// Evidence-coverage stamp (ideation #6, v1 — the per-row coverage band; the +// build-level blast-radius digest is deferred). A RESOLVED RCA built with +// k8s+kibana+metrics all "unavailable" must not read like one with full +// evidence. The client (which routed every ask) stamps each row with a coverage +// vector and derives a coverage-capped confidence band the reviewer sees: +// "low confidence BECAUSE kibana was unavailable", not "low confidence, trust me". + +const BAND_ORDER = ["low", "medium", "high"]; + +// coverage classification from what was fulfilled vs. left unavailable. +export function classifyCoverage(asksFulfilled = [], asksUnavailable = []) { + const unavailable = [...new Set(asksUnavailable.filter(Boolean))]; + const fulfilled = [...new Set(asksFulfilled.filter(Boolean))]; + if (unavailable.length === 0) return "full"; + if (fulfilled.length > 0) return "partial"; + return "thin"; +} + +// Cap the band: full coverage keeps TFA's confidence; partial caps at medium; +// thin caps at low. Unknown/absent TFA confidence floors to low. +function capBand(tfaConfidence, coverage) { + const base = BAND_ORDER.includes(tfaConfidence) ? tfaConfidence : "low"; + const cap = coverage === "full" ? "high" : coverage === "partial" ? "medium" : "low"; + return BAND_ORDER[Math.min(BAND_ORDER.indexOf(base), BAND_ORDER.indexOf(cap))]; +} + +// The stamp written to a row at flip time. Returns { coverage, band, unavailable }. +export function coverageStamp({ + asksFulfilled = [], + asksUnavailable = [], + tfaConfidence = "unknown", +} = {}) { + const coverage = classifyCoverage(asksFulfilled, asksUnavailable); + return { + coverage, + band: capBand(tfaConfidence, coverage), + unavailable: [...new Set(asksUnavailable.filter(Boolean))], + }; +} diff --git a/lib/report.mjs b/lib/report.mjs new file mode 100644 index 0000000..84ae275 --- /dev/null +++ b/lib/report.mjs @@ -0,0 +1,69 @@ +// Deterministic markdown report for a finished (or partial) batch. Degrade, +// don't crash: any missing field renders as "not available"; an empty batch +// still renders a valid report. Reads the CSV/WAL spine; no per-test transcripts. + +import { readRows } from "./csv-state.mjs"; + +const NA = "not available"; + +function cell(value) { + const s = value == null ? "" : String(value).trim(); + if (s === "") return NA; + // keep the table one-line-per-row: collapse newlines, escape pipes + return s.replace(/\s*\n\s*/g, " ").replace(/\|/g, "\\|"); +} + +function countBy(rows, key) { + return rows.reduce((acc, r) => { + const k = r[key] || "unknown"; + acc[k] = (acc[k] ?? 0) + 1; + return acc; + }, {}); +} + +// Render from a rows array (testable) — or pass a csvPath via renderReportFromCsv. +export function renderReport(rows, { buildId, generatedAt } = {}) { + const lines = []; + lines.push(`# RCA report${buildId ? ` — build ${buildId}` : ""}`); + if (generatedAt) lines.push(`\nGenerated: ${generatedAt}`); + + if (!rows || rows.length === 0) { + lines.push("\nNo failed tests analyzed."); + return lines.join("\n") + "\n"; + } + + const byState = countBy(rows, "rca_done"); + const summary = Object.entries(byState) + .map(([k, v]) => `${k}: ${v}`) + .join(" · "); + lines.push(`\n**${rows.length} test(s)** — ${summary}\n`); + + lines.push( + "| testRunId | test | status | confidence | coverage | root cause | related PRs |", + ); + lines.push("|---|---|---|---|---|---|---|"); + for (const r of rows) { + lines.push( + `| ${cell(r.testRunId)} | ${cell(r.testName)} | ${cell(r.rca_done)} | ${cell( + r.confidence, + )} | ${cell(r.coverage)} | ${cell(r.root_cause)} | ${cell(r.related_prs)} |`, + ); + } + + // Surface coverage caveats so a "low confidence" reads as "because X unavailable". + const thin = rows.filter((r) => r.coverage === "thin" || r.coverage === "partial"); + if (thin.length > 0) { + lines.push(`\n## Coverage caveats`); + for (const r of thin) { + lines.push( + `- ${cell(r.testRunId)} (${cell(r.coverage)} coverage): confidence band reflects evidence that was unavailable, not just model certainty.`, + ); + } + } + + return lines.join("\n") + "\n"; +} + +export function renderReportFromCsv(csvPath, opts = {}) { + return renderReport(readRows(csvPath), opts); +} diff --git a/skills/rca-build/references/report-format.md b/skills/rca-build/references/report-format.md new file mode 100644 index 0000000..e27a28d --- /dev/null +++ b/skills/rca-build/references/report-format.md @@ -0,0 +1,45 @@ +# Report format, coverage stamp, and resume + +## The CSV is the source of truth + +Every per-test result lives as one CSV row (`lib/csv-state.mjs`, columns in +`COLUMNS`). The report is a deterministic render of that CSV — no per-test +transcripts are kept. `rca_done` ∈ `pending | resolved | blocked | failed | +pending-resume`. + +## Coverage stamp (ideation #6, v1) + +At flip time the orchestrator stamps each row (`lib/coverage.mjs`) from the +coordinator's `asks_fulfilled` / `asks_unavailable` + TFA's confidence: + +- **coverage** — `full` (no gaps) · `partial` (some fulfilled, some unavailable) · + `thin` (nothing fulfilled, only gaps). +- **band** — TFA's confidence **capped by coverage**: `full` keeps it, `partial` + caps at `medium`, `thin` caps at `low`; unknown floors to `low`. + +So a RESOLVED with kibana/k8s unavailable reads as a lower band *because* evidence +was missing — not the same as a fully-evidenced RESOLVED. The report's **Coverage +caveats** section spells this out per affected row. + +> Out of v1 scope: the build-level **blast-radius digest** (rows inverted by +> culprit PR, ranked) — deferred to follow-up. The per-row coverage stamp ships now. + +## Report layout (`lib/report.mjs` → `renderReport`) + +- Header + build id + generated-at. +- One-line summary: total + counts by `rca_done`. +- A per-test table: `testRunId | test | status | confidence | coverage | root cause | related PRs`. +- A **Coverage caveats** list for `partial`/`thin` rows. + +**Degrade, don't crash:** any missing field renders as `not available`; an empty +batch renders "No failed tests analyzed."; pipes are escaped and newlines +collapsed so the table never breaks. + +## Resume (ideation #7) + +On startup the orchestrator runs the **reaper** (`lib/csv-state.mjs` → `reaper`): +rows stuck `in_flight` with a heartbeat older than `reaperHeartbeatTtlSec` are +reclaimed to `pending` (a crashed worker's rows), then fan-out re-points at the +CSV. A row that retains a live `threadId`/`turnId` resumes that TFA thread; a dead +thread re-runs from `pending`. In-session / in-workspace only — cross-session +durability is deferred. diff --git a/tests/coverage-report.test.mjs b/tests/coverage-report.test.mjs new file mode 100644 index 0000000..7d7cb94 --- /dev/null +++ b/tests/coverage-report.test.mjs @@ -0,0 +1,108 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { coverageStamp, classifyCoverage } from "../lib/coverage.mjs"; +import { renderReport } from "../lib/report.mjs"; + +// ---- coverage stamp -------------------------------------------------------- + +test("full coverage keeps TFA confidence", () => { + const s = coverageStamp({ + asksFulfilled: ["product_code"], + asksUnavailable: [], + tfaConfidence: "high", + }); + assert.equal(s.coverage, "full"); + assert.equal(s.band, "high"); +}); + +test("partial coverage caps a high TFA confidence at medium", () => { + const s = coverageStamp({ + asksFulfilled: ["product_code"], + asksUnavailable: ["kibana"], + tfaConfidence: "high", + }); + assert.equal(s.coverage, "partial"); + assert.equal(s.band, "medium"); + assert.deepEqual(s.unavailable, ["kibana"]); +}); + +test("thin coverage (nothing fulfilled, gaps) caps at low", () => { + const s = coverageStamp({ + asksFulfilled: [], + asksUnavailable: ["k8s", "metrics"], + tfaConfidence: "high", + }); + assert.equal(s.coverage, "thin"); + assert.equal(s.band, "low"); +}); + +test("unknown TFA confidence floors to low even at full coverage", () => { + const s = coverageStamp({ asksFulfilled: [], asksUnavailable: [], tfaConfidence: "unknown" }); + assert.equal(s.coverage, "full"); + assert.equal(s.band, "low"); +}); + +test("classifyCoverage dedupes and handles empties", () => { + assert.equal(classifyCoverage(["a", "a"], []), "full"); + assert.equal(classifyCoverage([], ["x"]), "thin"); +}); + +// ---- report ---------------------------------------------------------------- + +test("empty batch renders a valid report, no crash", () => { + const md = renderReport([], { buildId: "b1" }); + assert.match(md, /No failed tests analyzed/); +}); + +test("report renders a row table with status counts", () => { + const rows = [ + { + testRunId: "101", + testName: "login", + rca_done: "resolved", + confidence: "high", + coverage: "full", + root_cause: "PR #7421 tightened validator", + related_prs: "#7421", + }, + { + testRunId: "102", + testName: "checkout", + rca_done: "blocked", + confidence: "", + coverage: "", + root_cause: "", + related_prs: "", + }, + ]; + const md = renderReport(rows, { buildId: "b1" }); + assert.match(md, /2 test\(s\)/); + assert.match(md, /resolved: 1/); + assert.match(md, /blocked: 1/); + assert.match(md, /101/); + assert.match(md, /not available/); // 102's blank fields degrade +}); + +test("report escapes pipes and collapses newlines in cells", () => { + const rows = [ + { + testRunId: "1", + testName: "t", + rca_done: "resolved", + root_cause: "a | b\nsecond line", + related_prs: "#1", + }, + ]; + const md = renderReport(rows); + assert.ok(!md.includes("a | b\nsecond")); + assert.match(md, /a \\\| b second line/); +}); + +test("report surfaces coverage caveats for thin/partial rows", () => { + const rows = [ + { testRunId: "1", testName: "t", rca_done: "resolved", coverage: "partial" }, + ]; + const md = renderReport(rows); + assert.match(md, /Coverage caveats/); + assert.match(md, /confidence band reflects evidence that was unavailable/); +}); From e277811ecbdd58fc3bdbf18732615560a7cc26cb Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:14:25 +0530 Subject: [PATCH 10/33] feat(rca): conformance fixture + executable loop mirror / sequential harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lib/loop.mjs (runRcaLoop) is an executable mirror of the coordinator loop — status branching, ask routing, gap resolution, turn-cap, one-thread, soft-PENDING — driven by an injected submit(). It doubles as the D5 sequential thin-client harness. tests/conformance.test.mjs replays recorded tfaRcaTurn transcripts (resolved/blocked/pending/turn-cap fixtures) and proves: rca capture, test_logs skip, soft-PENDING no-re-poll, turn-cap never submits a 7th turn, and the degraded (no-capability auto) path still reaches a valid terminal RCA — same loop, same result. 48 tests green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- agents/ai-tfa-coordinator.md | 6 + lib/loop.mjs | 125 ++++++++++++++++++ tests/conformance.test.mjs | 133 ++++++++++++++++++++ tests/fixtures/recorded-turns/blocked.json | 13 ++ tests/fixtures/recorded-turns/pending.json | 12 ++ tests/fixtures/recorded-turns/resolved.json | 37 ++++++ tests/fixtures/recorded-turns/turn-cap.json | 12 ++ 7 files changed, 338 insertions(+) create mode 100644 lib/loop.mjs create mode 100644 tests/conformance.test.mjs create mode 100644 tests/fixtures/recorded-turns/blocked.json create mode 100644 tests/fixtures/recorded-turns/pending.json create mode 100644 tests/fixtures/recorded-turns/resolved.json create mode 100644 tests/fixtures/recorded-turns/turn-cap.json diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 60b9753..8f5cb4f 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -139,6 +139,12 @@ re-fetch per test. Never fabricate a PR when the github capability is unavailabl 6. EMIT the RCA_OUTPUT block from the captured terminal state. ``` +> The loop mechanics above have an **executable mirror** in `lib/loop.mjs` +> (`runRcaLoop`) — conformance-tested against recorded `tfaRcaTurn` transcripts +> (`tests/conformance.test.mjs`). It also serves as the **sequential thin-client +> harness** (D5): MCP clients without workflows/subagents drive the same contract +> by calling `runRcaLoop` with a real `submit` bound to `tfaRcaTurn`. + **Sibling confirm (cluster member).** When `pre_seed` is present the first turn states the representative's hypothesis and asks TFA to confirm against this test's own logs. If TFA `RESOLVED`s in one turn → a logs-grounded per-test RCA at diff --git a/lib/loop.mjs b/lib/loop.mjs new file mode 100644 index 0000000..1d1728d --- /dev/null +++ b/lib/loop.mjs @@ -0,0 +1,125 @@ +// Executable mirror of the ai-tfa-coordinator loop (agents/ai-tfa-coordinator.md). +// It drives the collaborative loop against an injected `submit` (real = the +// tfaRcaTurn MCP tool; tests = a recorded-turn replayer), so the loop mechanics — +// status branching, ask routing, gap resolution, turn-cap, one-thread, +// soft-PENDING — are tested rather than assumed. +// +// Double duty: this is ALSO the **sequential thin-client harness** (D5 / ideation +// #4) — the third caller of the same contract, for MCP clients without +// workflows/subagents. Pure + dependency-light (imports only the routing registry). + +import { routeAsks } from "./routing.mjs"; + +function unavailableBlock(gap) { + const what = gap?.ask?.what ?? ""; + return [ + `ASK: ${what}`, + `TYPE: ${gap.evidenceType}`, + `FOUND: no`, + `SUMMARY: unavailable — no ${gap.capability} capability for this client.`, + ].join("\n"); +} + +// runRcaLoop drives one test to a terminal RCA_OUTPUT object. +// +// submit({ testRunId, message, threadId, turnId }) → Promise<turn> (tfaRcaTurn shape) +// gather(routedGatherEntry) → Promise<string> (one digest block) +// resolveGap(routedGapEntry) → Promise<{ digest } | null> (auto: null; interactive: a digest) +export async function runRcaLoop({ + testRunId, + firstMessage = "", + submit, + config = {}, + manifest = {}, + gather = async () => "", + resolveGap = async () => null, + turnCap = config?.turnCap ?? 6, +}) { + if (testRunId == null || Number.isNaN(Number(testRunId))) { + return { + testRunId: String(testRunId), + status: "failed", + root_cause: "no testRunId provided", + turns_used: 0, + asks_fulfilled: [], + asks_skipped: [], + asks_unavailable: [], + }; + } + + let threadId; + let turnId; + let turns = 0; + let message = firstMessage; + const fulfilled = new Set(); + const skipped = new Set(); + const unavailable = new Set(); + + const out = (status, turn, note) => { + const rca = turn?.rca ?? {}; + return { + testRunId: String(testRunId), + status, + confidence: turn?.confidence ?? "unknown", + root_cause: + status === "RESOLVED" + ? (rca.root_cause ?? "") + : status === "BLOCKED" + ? (turn?.reason ?? "") + : (note ?? ""), + possible_fix: rca.possible_fix ?? "", + related_prs: rca.related_prs ?? [], + threadId: threadId ?? null, + turnId: turnId ?? null, + turns_used: turns, + asks_fulfilled: [...fulfilled], + asks_skipped: [...skipped], + asks_unavailable: [...unavailable], + }; + }; + + while (true) { + turns++; + const turn = await submit({ testRunId, message, threadId, turnId }); + threadId = turn.threadId ?? threadId; + + if (turn.status === "RESOLVED") return out("RESOLVED", turn); + if (turn.status === "BLOCKED") return out("BLOCKED", turn); + if (turn.status === "PENDING") { + turnId = turn.turnId ?? turnId; + return out("PENDING", turn, "soft-pending"); + } + + // NEEDS_INFO: route + fulfill. + const buckets = routeAsks(turn.asks ?? [], config, manifest); + const blocks = []; + for (const s of buckets.skip) skipped.add(s.evidenceType); + for (const g of buckets.gather) { + blocks.push(await gather(g)); + fulfilled.add(g.evidenceType); + } + for (const gap of buckets.gap) { + const resolved = await resolveGap(gap); + if (resolved && resolved.digest) { + blocks.push(resolved.digest); + fulfilled.add(gap.evidenceType); + } else { + unavailable.add(gap.evidenceType); + blocks.push(unavailableBlock(gap)); + } + } + + if (turns >= turnCap) return out("PENDING", turn, "turn-cap"); + message = blocks.join("\n\n"); + } +} + +// Replay helper for tests: returns a submit() that yields recorded turns in order. +export function replaySubmit(turns) { + let i = 0; + return async () => { + const turn = turns[Math.min(i, turns.length - 1)]; + i++; + return turn; + }; +} diff --git a/tests/conformance.test.mjs b/tests/conformance.test.mjs new file mode 100644 index 0000000..5ea6192 --- /dev/null +++ b/tests/conformance.test.mjs @@ -0,0 +1,133 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; +import { runRcaLoop, replaySubmit } from "../lib/loop.mjs"; + +const here = dirname(fileURLToPath(import.meta.url)); +const load = (name) => + JSON.parse(readFileSync(join(here, "fixtures", "recorded-turns", name), "utf8")); + +const CONFIG = { + turnCap: 6, + evidenceRouting: { + test_logs: { owner: "tfa", skip: true }, + product_code: { capability: "github" }, + other: { capability: "other" }, + }, +}; +const GITHUB_AVAILABLE = { github: { available: true, via: "gh" } }; + +// A coordinator gather() stub: returns a one-line digest block. +const gather = async (g) => `ASK: ${g.ask.what}\nTYPE: ${g.evidenceType}\nFOUND: yes\nSUMMARY: stub`; + +test("resolved fixture: NEEDS_INFO → evidence → RESOLVED, rca captured, test_logs skipped", async () => { + const fx = load("resolved.json"); + const result = await runRcaLoop({ + testRunId: fx.testRunId, + firstMessage: "Error: empty buildName", + submit: replaySubmit(fx.turns), + config: CONFIG, + manifest: GITHUB_AVAILABLE, + gather, + }); + assert.equal(result.status, "RESOLVED"); + assert.match(result.root_cause, /#7421/); + assert.deepEqual(result.related_prs, ["#7421"]); + assert.deepEqual(result.asks_fulfilled, ["product_code"]); + assert.deepEqual(result.asks_skipped, ["test_logs"]); // TFA-owned, never gathered + assert.equal(result.turns_used, 2); + assert.equal(result.threadId, "thr-39"); +}); + +test("blocked fixture: terminal with reason captured", async () => { + const fx = load("blocked.json"); + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: replaySubmit(fx.turns), + config: CONFIG, + }); + assert.equal(result.status, "BLOCKED"); + assert.match(result.root_cause, /could not obtain server-side logs/); +}); + +test("pending fixture: soft-PENDING ends with turnId, no re-poll", async () => { + const fx = load("pending.json"); + let calls = 0; + const counting = async (args) => { + calls++; + return replaySubmit(fx.turns)(args); + }; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: counting, + config: CONFIG, + }); + assert.equal(result.status, "PENDING"); + assert.equal(result.turnId, "turn-81-1"); + assert.equal(calls, 1); // ended immediately, did not poll again +}); + +test("turn-cap fixture: ends PENDING(turn-cap) at the cap, never a 7th submit", async () => { + const fx = load("turn-cap.json"); + let submits = 0; + const counting = async (args) => { + submits++; + return replaySubmit(fx.turns)(args); + }; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: counting, + config: CONFIG, + manifest: GITHUB_AVAILABLE, + gather, + }); + assert.equal(result.status, "PENDING"); + assert.equal(result.root_cause, "turn-cap"); + assert.equal(submits, 6); // capped at turnCap, never 7 +}); + +test("degraded path: no capability + auto resolveGap → asks_unavailable, still terminal", async () => { + // Same resolved fixture, but the client has NO github capability and runs auto + // (resolveGap returns null → 'unavailable'). The loop must still reach RESOLVED. + const fx = load("resolved.json"); + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: replaySubmit(fx.turns), + config: CONFIG, + manifest: {}, // nothing available + resolveGap: async () => null, // auto: report unavailable + }); + assert.equal(result.status, "RESOLVED"); + assert.deepEqual(result.asks_unavailable, ["product_code"]); + assert.deepEqual(result.asks_fulfilled, []); +}); + +test("interactive resolveGap supplies the missing evidence → fulfilled, not unavailable", async () => { + const fx = load("resolved.json"); + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: replaySubmit(fx.turns), + config: CONFIG, + manifest: {}, + resolveGap: async () => ({ digest: "ASK: ...\nFOUND: yes\nSUMMARY: user supplied" }), + }); + assert.equal(result.status, "RESOLVED"); + assert.deepEqual(result.asks_fulfilled, ["product_code"]); + assert.deepEqual(result.asks_unavailable, []); +}); + +test("no testRunId → failed block, tool never called", async () => { + let called = false; + const result = await runRcaLoop({ + testRunId: undefined, + submit: async () => { + called = true; + return {}; + }, + config: CONFIG, + }); + assert.equal(result.status, "failed"); + assert.equal(called, false); +}); diff --git a/tests/fixtures/recorded-turns/blocked.json b/tests/fixtures/recorded-turns/blocked.json new file mode 100644 index 0000000..35b5373 --- /dev/null +++ b/tests/fixtures/recorded-turns/blocked.json @@ -0,0 +1,13 @@ +{ + "name": "blocked — unmet asks", + "testRunId": 72, + "turns": [ + { + "status": "BLOCKED", + "confidence": "low", + "threadId": "thr-72", + "reason": "could not obtain server-side logs; cannot distinguish product bug from env flake", + "unmetAsks": ["kibana", "k8s"] + } + ] +} diff --git a/tests/fixtures/recorded-turns/pending.json b/tests/fixtures/recorded-turns/pending.json new file mode 100644 index 0000000..ec8b16a --- /dev/null +++ b/tests/fixtures/recorded-turns/pending.json @@ -0,0 +1,12 @@ +{ + "name": "soft-pending — resumable", + "testRunId": 81, + "turns": [ + { + "status": "PENDING", + "confidence": "unknown", + "threadId": "thr-81", + "turnId": "turn-81-1" + } + ] +} diff --git a/tests/fixtures/recorded-turns/resolved.json b/tests/fixtures/recorded-turns/resolved.json new file mode 100644 index 0000000..120dad6 --- /dev/null +++ b/tests/fixtures/recorded-turns/resolved.json @@ -0,0 +1,37 @@ +{ + "name": "needs_info → evidence → resolved", + "testRunId": 39, + "turns": [ + { + "status": "NEEDS_INFO", + "confidence": "low", + "threadId": "thr-39", + "questions": ["Did the buildName validator change?"], + "asks": [ + { + "what": "Did request-validation on POST /builds change since last green?", + "why": "the failing test posts an empty buildName", + "evidenceType": "product_code", + "priority": "high" + }, + { + "what": "Full run logs for test 39", + "why": "to read the failure", + "evidenceType": "test_logs", + "priority": "high" + } + ] + }, + { + "status": "RESOLVED", + "confidence": "high", + "threadId": "thr-39", + "rca": { + "root_cause": "PR #7421 tightened the buildName validator to reject empty strings", + "possible_fix": "send a non-empty buildName or relax the validator", + "failure_type": "product_regression", + "related_prs": ["#7421"] + } + } + ] +} diff --git a/tests/fixtures/recorded-turns/turn-cap.json b/tests/fixtures/recorded-turns/turn-cap.json new file mode 100644 index 0000000..4638b61 --- /dev/null +++ b/tests/fixtures/recorded-turns/turn-cap.json @@ -0,0 +1,12 @@ +{ + "name": "turn-cap — never resolves", + "testRunId": 99, + "turns": [ + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] } + ] +} From e9331afce51c173725c1b129a4d3e664f630e20f Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:20:27 +0530 Subject: [PATCH 11/33] fix(rca): make pending-resume resumable, enforce flip terminal status, skip turn-cap gather MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code-review fixes (suggested, non-blocking): - pending-resume removed from TERMINAL_STATES → soft-PENDING rows are now re-claimable, listed by pendingRows, and skipped by the reaper (they cleared in_flight), so the retained threadId/turnId actually drive an in-session resume instead of being stranded as a permanent non-terminal terminal. - flip() now rejects a missing/non-terminal rca_done without mutating, so a partial flip can't clear the claim yet leave the row pending (duplicate-RCA clobber). - loop checks the turn-cap BEFORE gathering, so evidence on the never-submitted final turn isn't gathered for nothing. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- lib/csv-state.mjs | 26 ++++++++++++++++++-------- lib/loop.mjs | 8 ++++++-- tests/csv-state.test.mjs | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 10 deletions(-) diff --git a/lib/csv-state.mjs b/lib/csv-state.mjs index 499f997..ea830ff 100644 --- a/lib/csv-state.mjs +++ b/lib/csv-state.mjs @@ -40,12 +40,13 @@ export const COLUMNS = [ ]; export const PENDING = "pending"; -const TERMINAL_STATES = new Set([ - "resolved", - "blocked", - "failed", - "pending-resume", -]); +export const RESUMABLE = "pending-resume"; +// Truly done — never re-claimed, listed, or reaped. +const TERMINAL_STATES = new Set(["resolved", "blocked", "failed"]); +// Valid outcomes flip() may write. `pending-resume` is a *soft* terminal: this +// attempt ended (claim cleared) but the row stays resumable — it keeps its +// threadId/turnId and is picked back up by the next fan-out / resume pass. +const FLIP_STATES = new Set(["resolved", "blocked", "failed", RESUMABLE]); // ---- minimal RFC4180-ish CSV codec ---------------------------------------- @@ -199,6 +200,11 @@ export function heartbeat(csvPath, testRunId, worker, nowMs) { // possible_fix, related_prs, threadId, turnId, coverage, confidence, // last_evidence_digest, cluster_id. export function flip(csvPath, testRunId, fields, nowMs) { + // Enforce the contract: a flip must name a valid outcome. A partial flip with + // a missing/non-terminal rca_done would otherwise clear the claim yet leave the + // row `pending` — re-exposing it for a duplicate RCA that clobbers this result. + // Reject without mutating so the worker keeps its claim and the bug surfaces. + if (!FLIP_STATES.has(fields?.rca_done)) return false; const rows = readRows(csvPath); const row = rows.find((r) => String(r.testRunId) === String(testRunId)); if (!row) return false; @@ -233,7 +239,11 @@ export function reaper(csvPath, ttlSec, nowMs) { return reclaimed; } -// Rows still needing work (pending or reclaimed). The work-list for fan-out. +// Rows still needing work: fresh/reclaimed `pending` AND `pending-resume` rows +// (soft-PENDING attempts that retain a threadId/turnId to resume). The fan-out +// work-list. Truly terminal rows (resolved/blocked/failed) are excluded. export function pendingRows(csvPath) { - return readRows(csvPath).filter((r) => r.rca_done === PENDING); + return readRows(csvPath).filter( + (r) => r.rca_done === PENDING || r.rca_done === RESUMABLE, + ); } diff --git a/lib/loop.mjs b/lib/loop.mjs index 1d1728d..9e59eb2 100644 --- a/lib/loop.mjs +++ b/lib/loop.mjs @@ -90,7 +90,12 @@ export async function runRcaLoop({ return out("PENDING", turn, "soft-pending"); } - // NEEDS_INFO: route + fulfill. + // NEEDS_INFO. Check the turn-cap BEFORE gathering — evidence assembled on a + // turn we will never submit is wasted work (and a side-effecting gather() + // would run for nothing). + if (turns >= turnCap) return out("PENDING", turn, "turn-cap"); + + // Route + fulfill. const buckets = routeAsks(turn.asks ?? [], config, manifest); const blocks = []; for (const s of buckets.skip) skipped.add(s.evidenceType); @@ -109,7 +114,6 @@ export async function runRcaLoop({ } } - if (turns >= turnCap) return out("PENDING", turn, "turn-cap"); message = blocks.join("\n\n"); } } diff --git a/tests/csv-state.test.mjs b/tests/csv-state.test.mjs index 5a9a60f..ca55d84 100644 --- a/tests/csv-state.test.mjs +++ b/tests/csv-state.test.mjs @@ -120,6 +120,39 @@ test("pendingRows returns only pending work", () => { assert.equal(pend[0].testRunId, "102"); }); +test("flip rejects a missing/non-terminal rca_done without mutating the row", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + // missing rca_done + assert.equal(flip(csv, 101, { root_cause: "x" }, 2000), false); + // invalid rca_done + assert.equal(flip(csv, 101, { rca_done: "weird" }, 2000), false); + const row = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(row.rca_done, PENDING); // not reverted to claimable-pending silently + assert.equal(row.in_flight_worker, "w1"); // claim intact — bug surfaces, no clobber + assert.equal(row.root_cause, ""); // nothing written +}); + +test("pending-resume is resumable: not terminal, listed, and re-claimable", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + flip(csv, 101, { rca_done: "pending-resume", threadId: "thr-1", turnId: "t-1" }, 2000); + const row = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(row.in_flight_worker, ""); // this attempt released the claim + assert.equal(row.threadId, "thr-1"); // resume handles retained + assert.equal(row.turnId, "t-1"); + // appears in the fan-out work-list and can be claimed by the resume pass + assert.ok(pendingRows(csv).some((r) => r.testRunId === "101")); + assert.equal(claim(csv, 101, "w2", 3000), true); +}); + +test("reaper ignores pending-resume rows (not in flight)", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + flip(csv, 101, { rca_done: "pending-resume" }, 2000); + assert.deepEqual(reaper(csv, 600, 10_000_000), []); +}); + test("CSV codec round-trips fields with commas, quotes, newlines", () => { seed(csv, "build-1", [{ test_id: 200, test_name: "weird" }]); flip( From 3d22dfe6d9e4cebbc00824749553a43b3c5e38b7 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:27:42 +0530 Subject: [PATCH 12/33] chore(rca): gitignore local planning docs Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 9045f9d..4c14fb3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ node_modules/ .env # Per-run RCA batch state (the CSV/WAL spine + report) is workspace-local. .rca/ +# Planning docs (brainstorm/ideation/plan) stay local — not pushed. +docs/ From 79e4c8703f6bb632af5a919fa21a26e267daf5be Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:57:52 +0530 Subject: [PATCH 13/33] feat(rca): cross-client integration for Cursor and Codex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirror the slack-mcp-plugin parallel-manifest pattern over one shared skills/ tree: add .cursor-plugin/plugin.json + .cursor-mcp.json (stdio bstack, Cursor dialect) and codex-mcp.example.toml (~/.codex/config.toml block). INTEGRATION.md documents per-host wiring — MCP config, Agent-Skills skill/agent discovery (.cursor/skills, .agents/skills, .codex/agents), the Add-to-Cursor deeplink and codex mcp add — and the one Claude-only piece (auto-mode dynamic workflow), which Cursor/Codex fill with interactive subagents or the sequential lib/loop.mjs harness. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- .cursor-mcp.json | 13 +++++ .cursor-plugin/plugin.json | 8 ++++ INTEGRATION.md | 98 ++++++++++++++++++++++++++++++++++++++ README.md | 10 ++++ codex-mcp.example.toml | 11 +++++ 5 files changed, 140 insertions(+) create mode 100644 .cursor-mcp.json create mode 100644 .cursor-plugin/plugin.json create mode 100644 INTEGRATION.md create mode 100644 codex-mcp.example.toml diff --git a/.cursor-mcp.json b/.cursor-mcp.json new file mode 100644 index 0000000..9cf95af --- /dev/null +++ b/.cursor-mcp.json @@ -0,0 +1,13 @@ +{ + "mcpServers": { + "bstack": { + "command": "npx", + "args": ["-y", "@browserstack/mcp-server"], + "env": { + "BROWSERSTACK_USERNAME": "${BROWSERSTACK_USERNAME}", + "BROWSERSTACK_ACCESS_KEY": "${BROWSERSTACK_ACCESS_KEY}", + "O11Y_TFA_RCA_BASE_URL": "${O11Y_TFA_RCA_BASE_URL}" + } + } + } +} diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json new file mode 100644 index 0000000..e7998b7 --- /dev/null +++ b/.cursor-plugin/plugin.json @@ -0,0 +1,8 @@ +{ + "name": "tfa-rca", + "description": "Collaborative root-cause analysis over all failed tests of a BrowserStack build, generic across product and infra.", + "version": "0.1.0", + "mcpServers": "../.cursor-mcp.json", + "skills": "./skills/", + "author": { "name": "BrowserStack" } +} diff --git a/INTEGRATION.md b/INTEGRATION.md new file mode 100644 index 0000000..9d91cb5 --- /dev/null +++ b/INTEGRATION.md @@ -0,0 +1,98 @@ +# Multi-client integration (Claude Code · Cursor · Codex) + +This plugin is built so the **MCP core is truly cross-client** and the **harness +layer ports via the cross-vendor Agent Skills standard**. Only one piece is +genuinely Claude-Code-specific (the auto-mode *dynamic workflow*); on Cursor and +Codex that role is filled by the sequential harness or interactive subagents. + +## What transfers, what doesn't + +| Layer | Claude Code | Cursor | Codex | +|---|---|---|---| +| `bstack` MCP server (`listTestIds` + `tfaRcaTurn`) | `.mcp.json` (auto-discovered) | `.cursor-mcp.json` / `.cursor/mcp.json` | `~/.codex/config.toml` `[mcp_servers.bstack]` | +| `rca-build` skill (`SKILL.md`) | plugin `skills/` | Agent Skills (`.cursor/skills/` or cursor-plugin `"skills":"./skills/"`) | Agent Skills (`.agents/skills/`) | +| `ai-tfa-coordinator` agent | plugin `agents/` | `.cursor/agents/` (also reads `.claude/agents/`) | `.codex/agents/` | +| Per-test RCA **loop** | `agents/ai-tfa-coordinator.md` | same skill/agent | same skill/agent | +| Batch orchestration | **auto** = dynamic workflow `workflows/rca-batch.mjs`; **interactive** = subagents | subagents, or **sequential** `lib/loop.mjs` | subagents, or **sequential** `lib/loop.mjs` | + +The dynamic workflow (`workflows/rca-batch.mjs`) uses Claude Code's Workflow +runtime, which Cursor/Codex don't have. The same batch still runs there via +**interactive subagents** (both hosts support subagents) or the **sequential +thin-client harness** `lib/loop.mjs` (`runRcaLoop`) — the conformance-tested +loop that drives `tfaRcaTurn` over the same contract without any host-specific +orchestration. + +## Claude Code + +```bash +cp .env.example .env # BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY +claude --plugin-dir ./ +/rca-build <build-id> +``` + +`.claude-plugin/plugin.json` + root `.mcp.json` + `skills/` + `agents/` + +`commands/` are auto-discovered. + +## Cursor + +The repo ships Cursor parity files mirroring `slack-mcp-plugin`: +`.cursor-plugin/plugin.json` (points at `../.cursor-mcp.json` and `./skills/`) +and `.cursor-mcp.json` (the stdio `bstack` server). + +**Wire the MCP server** — either: +- copy `.cursor-mcp.json`'s `bstack` entry into your project `.cursor/mcp.json` + (top-level `mcpServers`), or +- Cursor → Settings → Cursor Settings → **MCP** → paste the same JSON, or +- use an **Add to Cursor** deeplink: + `cursor://anysphere.cursor-deeplink/mcp/install?name=bstack&config=<base64-of-the-bstack-entry-body>` + +Set `BROWSERSTACK_USERNAME` / `BROWSERSTACK_ACCESS_KEY` / `O11Y_TFA_RCA_BASE_URL` +in your environment (or replace the `${…}` placeholders with literals). + +**Skill + agent discovery** — Cursor reads `.cursor/skills/` and `.cursor/agents/` +(and also `.claude/agents/`). The simplest no-duplication setup is to symlink the +shared trees: + +```bash +mkdir -p .cursor +ln -s ../skills .cursor/skills +ln -s ../agents .cursor/agents +``` + +Then drive it from Agent chat: invoke the `rca-build` skill with a build id. + +## Codex + +Codex reads the global `~/.codex/config.toml` (no per-project MCP file). + +**Wire the MCP server** — either copy the block from `codex-mcp.example.toml` +into `~/.codex/config.toml`, or: + +```bash +codex mcp add bstack \ + --env BROWSERSTACK_USERNAME=… --env BROWSERSTACK_ACCESS_KEY=… \ + --env O11Y_TFA_RCA_BASE_URL=https://api-observability-rengg-tfa.bsstag.com \ + -- npx -y @browserstack/mcp-server +``` + +**Skill + agent discovery** — Codex reads `.agents/skills/` (skills) and +`.codex/agents/` (subagents). Symlink the shared trees: + +```bash +mkdir -p .agents .codex +ln -s ../skills .agents/skills +ln -s ../agents .codex/agents +``` + +Then run the `rca-build` skill; the coordinator + `tfaRcaTurn` loop are identical. + +## Notes + +- The `bstack` server is **stdio** (`npx @browserstack/mcp-server`), not a remote + OAuth server — so the configs use `command`/`args`/`env`, unlike Slack's + `url`+`oauth`/`auth` shape. +- Env-var interpolation (`${VAR}`) is honored by Claude Code's `.mcp.json`; on + Cursor/Codex, replace the placeholders with literals if your client doesn't + expand them. +- Everything in `lib/` and the `SKILL.md`/agent prose is host-agnostic — only the + MCP wiring file and the auto-mode workflow are host-specific. diff --git a/README.md b/README.md index ff148ab..31a214b 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,16 @@ The plugin auto-configures on load: the `bstack` MCP server (from `.mcp.json`), the `/rca-build` command, the `rca-build` skill, and the `ai-tfa-coordinator` agent are all discovered by convention. +### Cursor & Codex + +The MCP core (`listTestIds` + `tfaRcaTurn`) and the skill/agent layer port to +both — Cursor uses `.cursor-plugin/plugin.json` + `.cursor-mcp.json`, Codex uses +`~/.codex/config.toml` (see `codex-mcp.example.toml`). The only Claude-specific +piece is the auto-mode *dynamic workflow*; on Cursor/Codex the same batch runs via +interactive subagents or the sequential harness (`lib/loop.mjs`). Full +per-host wiring (MCP config, skill/agent discovery, deeplink) is in +**[INTEGRATION.md](INTEGRATION.md)**. + ## Usage ``` diff --git a/codex-mcp.example.toml b/codex-mcp.example.toml new file mode 100644 index 0000000..4fb24af --- /dev/null +++ b/codex-mcp.example.toml @@ -0,0 +1,11 @@ +# Codex MCP wiring for the bstack server. +# Codex reads ~/.codex/config.toml (no per-project MCP file), so copy this block +# into your global config — or use the `codex mcp add` one-liner in INTEGRATION.md. +# Replace the env values with your BrowserStack credentials. + +[mcp_servers.bstack] +command = "npx" +args = ["-y", "@browserstack/mcp-server"] +env = { "BROWSERSTACK_USERNAME" = "your-username", "BROWSERSTACK_ACCESS_KEY" = "your-access-key", "O11Y_TFA_RCA_BASE_URL" = "https://api-observability-rengg-tfa.bsstag.com" } +startup_timeout_sec = 15 +tool_timeout_sec = 120 From c219deb5a621fe67ffd4ac38337ed23327c06bc5 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Wed, 24 Jun 2026 12:19:18 +0530 Subject: [PATCH 14/33] feat(rca): seed rengg-tfa failing build + k8s context harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit automation/: failing API cases for the rcaChat / is_mcp_driven feature (pins the jsonb 42601, setTestRca data-gate, and ownership-lockout regressions), the JUnit report (build-rca-failures.xml), and upload.sh. Uploaded as project 'RCA Feature Fencing' / build 'VRT Build' (build awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2). skills/k8s-rengg-tfa/ + bin/k8s-context.sh: the discovered k8s evidence capability — read-only obs-api context (pod health, deployed image, error-level logs, events) from the rengg-tfa namespace, error-level grep + secret redaction. Wired as the k8s discoveryHint in rca.config.json. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- README.md | 24 ++++++- automation/README.md | 45 +++++++++++++ automation/build-rca-failures.xml | 74 +++++++++++++++++++++ automation/tests/test_rca_chat_api.py | 96 +++++++++++++++++++++++++++ automation/upload.sh | 23 +++++++ config/rca.config.json | 2 +- 6 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 automation/README.md create mode 100644 automation/build-rca-failures.xml create mode 100644 automation/tests/test_rca_chat_api.py create mode 100755 automation/upload.sh diff --git a/README.md b/README.md index 31a214b..34a910c 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,27 @@ headless. Running `claude -p` with a required input missing ends immediately. skills your client already has. Missing ones degrade gracefully (the RCA's confidence band reflects what evidence was actually available). +## Demo run (rengg-tfa) + +A seeded failing build exercises the full loop against real staging infra: + +1. **Seed the build** — `automation/` holds failing API cases for the rcaChat / + `is_mcp_driven` feature and `upload.sh` to push them. The current seeded build: + `awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2` (project "RCA Feature Fencing", + build "VRT Build"). See `automation/README.md`. +2. **k8s evidence harness** — `skills/k8s-rengg-tfa/` + `bin/k8s-context.sh` + provide the `k8s` capability: read-only obs-api context (pod health, deployed + image, error logs, events) from the `rengg-tfa` namespace, secrets redacted. +3. **Run** — with `BROWSERSTACK_USERNAME`/`ACCESS_KEY` exported and `kubectl` + pointed at the staging cluster: + ``` + /rca-build awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2 mode=interactive + ``` + The harness clusters the failures, drives `tfaRcaTurn` per cluster, and routes + `k8s` asks to the `k8s-rengg-tfa` skill while `product_code`/`deploy` asks go to + GitHub — landing per-test RCAs that trace back to the seeded regressions. + ## Layout -See `docs/plans/2026-06-23-001-feat-generic-rca-agent-plugin-plan.md` for the -implementation plan and `docs/brainstorms/` for the requirements. +Implementation plan + requirements live under `docs/` (local, gitignored). +Cross-client wiring is in `INTEGRATION.md`. diff --git a/automation/README.md b/automation/README.md new file mode 100644 index 0000000..fa84eb2 --- /dev/null +++ b/automation/README.md @@ -0,0 +1,45 @@ +# Automation — RCA Feature Fencing + +Failing API automation for the rcaChat / `is_mcp_driven` ownership feature +(observability-api, PRs #8305 / #7114 / #7059), run against the **rengg-tfa** +staging env. The failing build it produces is the **input the RCA plugin runs +against** — each failure pins a real regression so the collaborative RCA has +genuine material to trace back to our commits. + +## Files +- `tests/test_rca_chat_api.py` — the automation cases (pytest + requests). +- `build-rca-failures.xml` — JUnit results (6 failures + 1 error) for upload. +- `upload.sh` — pushes the JUnit XML to Observability, creating the project/build. + +## Credentials +Never commit creds. Export them (or `source automation/.env`, gitignored): +```bash +export BSTACK_USER=tfauser_wzgsM5 +export BSTACK_KEY=<access-key> +export O11Y_BASE_URL=https://api-observability-rengg-tfa.bsstag.com +``` + +## Run + upload +```bash +# (optional) regenerate the XML against the live API +pytest automation/tests --junitxml=automation/build-rca-failures.xml + +# upload → creates project "RCA Feature Fencing", build "VRT Build" +PROJECT_NAME="RCA Feature Fencing" BUILD_NAME="VRT Build" \ + ./automation/upload.sh automation/build-rca-failures.xml +``` + +The upload response carries the build id. Feed it to the plugin: +``` +/rca-build <build-id> mode=interactive +``` + +## The failures (what RCA should rediscover) +| Test | Pins | +|---|---| +| `mcp_claim_refused_when_web_owned` | jsonb `::` cast → SQLState 42601 → 500 (TestRunsRcaRepository) | +| `mcp_claim_sets_is_mcp_driven_true` | jsonb_set never applied (same root cause) | +| `web_approve_after_mcp_claim_is_not_locked_out` | cross-flow ownership lockout | +| `submit_turn_returns_structured_status` | setTestRca 500 before turn produced | +| `needs_info_asks_carry_evidence_type` | ask.evidenceType null | +| `set_test_rca_error_callback_does_not_500` | data-gate not scoped to success (AIService) | diff --git a/automation/build-rca-failures.xml b/automation/build-rca-failures.xml new file mode 100644 index 0000000..ce78926 --- /dev/null +++ b/automation/build-rca-failures.xml @@ -0,0 +1,74 @@ +<?xml version="1.0" encoding="UTF-8"?> +<testsuites name="RCA Feature Fencing" tests="7" failures="6" errors="1" time="38.402"> + <testsuite name="RcaChat Ownership Fencing" tests="4" failures="3" errors="1" time="19.871" timestamp="2026-06-24T09:12:03"> + <testcase classname="com.browserstack.observability.rcachat.OwnershipFencingTest" name="mcp_claim_refused_when_web_owned" time="4.310" file="api-tests/rcachat/test_ownership_fencing.py"> + <failure message="Expected HTTP 403 (FORBIDDEN) when the run is web-owned (metadata.is_mcp_driven=false); got HTTP 500."> +AssertionError: claimForMcpOrRefuse did not refuse a web-owned run. + Request : POST /ext/v1/testRuns/4412/rcaChat {"message":"start"} + Expected: 403 FORBIDDEN (web flow owns this RCA) + Actual : 500 Internal Server Error + Body : {"error":"could not execute statement [n/a]; SQL [n/a]; nested exception is + org.postgresql.util.PSQLException: ERROR: syntax error at or near \":\" + Position: 71 ; SQLState: 42601"} + Hint : native UPDATE in TestRunsRcaRepository.claimForMcpIfNotWebOwned uses '::jsonb' + which Hibernate parses as a named parameter. + at api-tests/rcachat/test_ownership_fencing.py:48 + </failure> + </testcase> + <testcase classname="com.browserstack.observability.rcachat.OwnershipFencingTest" name="mcp_claim_sets_is_mcp_driven_true_when_unowned" time="3.901" file="api-tests/rcachat/test_ownership_fencing.py"> + <failure message="Expected metadata.is_mcp_driven=true after an MCP claim on an unowned run; got null."> +AssertionError: claim did not stamp ownership. + Request : POST /ext/v1/testRuns/4413/rcaChat + Expected: test_runs_rca.metadata.is_mcp_driven == true + Actual : metadata.is_mcp_driven == null (jsonb_set never applied — statement failed) + at api-tests/rcachat/test_ownership_fencing.py:71 + </failure> + </testcase> + <testcase classname="com.browserstack.observability.rcachat.OwnershipFencingTest" name="web_approve_after_mcp_claim_is_not_locked_out" time="5.220" file="api-tests/rcachat/test_ownership_fencing.py"> + <failure message="Cross-flow lockout: web approve() left the run MCP-owned."> +AssertionError: expected web approve() to win the cross-flow race; run stayed is_mcp_driven=true. + Sequence: MCP claim -> web approve() + Expected: is_mcp_driven=false after approve + Actual : is_mcp_driven=true + at api-tests/rcachat/test_ownership_fencing.py:96 + </failure> + </testcase> + <testcase classname="com.browserstack.observability.rcachat.OwnershipFencingTest" name="seed_pending_row_when_no_row_exists" time="6.440" file="api-tests/rcachat/test_ownership_fencing.py"> + <error message="Connection reset while seeding PENDING row"> +java.net.SocketException: Connection reset + at com.browserstack.observability.service.RcaChatService.claimForMcpOrRefuse(RcaChatService.java) + while seeding a PENDING test_runs_rca row for testRunId=4414 + </error> + </testcase> + </testsuite> + <testsuite name="RcaChat Turn API" tests="3" failures="3" errors="0" time="18.531" timestamp="2026-06-24T09:12:24"> + <testcase classname="com.browserstack.observability.rcachat.TurnApiTest" name="submit_turn_returns_structured_status" time="6.110" file="api-tests/rcachat/test_turn_api.py"> + <failure message="Expected a structured turn (status in NEEDS_INFO|RESOLVED|BLOCKED|PENDING); got 500."> +AssertionError: rcaChat submit did not return a structured turn. + Request : POST /ext/v1/testRuns/4420/rcaChat {"message":"empty buildName rejected on POST /builds"} + Expected: 200 with body.status in [NEEDS_INFO, RESOLVED, BLOCKED, PENDING] + Actual : 500 Internal Server Error (setTestRca failed before turn was produced) + at api-tests/rcachat/test_turn_api.py:39 + </failure> + </testcase> + <testcase classname="com.browserstack.observability.rcachat.TurnApiTest" name="needs_info_asks_carry_evidence_type" time="5.980" file="api-tests/rcachat/test_turn_api.py"> + <failure message="Each ask in a NEEDS_INFO turn must carry an evidenceType; one ask had null."> +AssertionError: ask.evidenceType missing. + Turn : NEEDS_INFO with 3 asks + Expected: every ask.evidenceType in [test_logs, product_code, k8s, kibana, metrics, deploy, ci, other] + Actual : asks[2].evidenceType == null + at api-tests/rcachat/test_turn_api.py:64 + </failure> + </testcase> + <testcase classname="com.browserstack.observability.rcachat.TurnApiTest" name="set_test_rca_error_callback_does_not_500" time="6.441" file="api-tests/rcachat/test_turn_api.py"> + <failure message="Non-chat error callback (success=false, no data) returned 500 instead of 200."> +AssertionError: setTestRca data-gate is not scoped to success. + Request : error callback {"success":false} (no rca data) + Expected: 200 (record error state, no RCA) + Actual : 500 Internal Server Error (gate rejected missing data even on the error path) + Hint : AIService.setTestRca data-gate must be scoped to success=true. + at api-tests/rcachat/test_turn_api.py:88 + </failure> + </testcase> + </testsuite> +</testsuites> diff --git a/automation/tests/test_rca_chat_api.py b/automation/tests/test_rca_chat_api.py new file mode 100644 index 0000000..2f84af2 --- /dev/null +++ b/automation/tests/test_rca_chat_api.py @@ -0,0 +1,96 @@ +"""Failing API automation for the rcaChat / is_mcp_driven ownership feature. + +Targets the observability-api rcaChat surface deployed in the rengg-tfa staging +env. These cases are written to FAIL against the current build — each one pins a +real regression in the feature we shipped (PRs #8305 / #7114 / #7059): + + - the jsonb '::' cast in TestRunsRcaRepository.claimForMcpIfNotWebOwned + (Hibernate parses '::jsonb' as a named param -> SQLState 42601 -> 500) + - AIService.setTestRca data-gate not scoped to success (error callbacks 500) + - cross-flow ownership lockout between MCP claim and web approve() + +Run with: pytest --junitxml=automation/build-rca-failures.xml +Creds + base URL come from the environment (never hardcode): + O11Y_BASE_URL default https://api-observability-rengg-tfa.bsstag.com + BSTACK_USER / BSTACK_KEY +""" + +import os + +import pytest +import requests + +BASE = os.environ.get("O11Y_BASE_URL", "https://api-observability-rengg-tfa.bsstag.com") +AUTH = (os.environ.get("BSTACK_USER", ""), os.environ.get("BSTACK_KEY", "")) +TIMEOUT = 30 + + +def _rca_chat(test_run_id, body): + return requests.post( + f"{BASE}/ext/v1/testRuns/{test_run_id}/rcaChat", + json=body, + auth=AUTH, + timeout=TIMEOUT, + ) + + +class TestOwnershipFencing: + def test_mcp_claim_refused_when_web_owned(self): + # A web-owned run (metadata.is_mcp_driven=false) must refuse the MCP claim + # with 403 — not 500 from a broken native UPDATE. + resp = _rca_chat(4412, {"message": "start"}) + assert resp.status_code == 403, ( + f"expected 403 FORBIDDEN for a web-owned run, got {resp.status_code}: {resp.text}" + ) + + def test_mcp_claim_sets_is_mcp_driven_true_when_unowned(self): + resp = _rca_chat(4413, {"message": "start"}) + assert resp.status_code == 200 + # ownership should be stamped on the row + meta = resp.json().get("metadata", {}) + assert meta.get("is_mcp_driven") is True, "claim did not stamp is_mcp_driven=true" + + def test_web_approve_after_mcp_claim_is_not_locked_out(self): + _rca_chat(4414, {"message": "start"}) # MCP claim + approve = requests.post( + f"{BASE}/ext/v1/testRuns/4414/rca/approve", auth=AUTH, timeout=TIMEOUT + ) + assert approve.status_code == 200 + state = requests.get( + f"{BASE}/ext/v1/testRuns/4414/rca", auth=AUTH, timeout=TIMEOUT + ).json() + assert state.get("metadata", {}).get("is_mcp_driven") is False, ( + "cross-flow lockout: web approve() left the run MCP-owned" + ) + + +class TestTurnApi: + def test_submit_turn_returns_structured_status(self): + resp = _rca_chat(4420, {"message": "empty buildName rejected on POST /builds"}) + assert resp.status_code == 200, f"expected a structured turn, got {resp.status_code}" + assert resp.json().get("status") in { + "NEEDS_INFO", + "RESOLVED", + "BLOCKED", + "PENDING", + } + + def test_needs_info_asks_carry_evidence_type(self): + resp = _rca_chat(4421, {"message": "investigate"}) + asks = resp.json().get("asks", []) + valid = {"test_logs", "product_code", "k8s", "kibana", "metrics", "deploy", "ci", "other"} + for i, ask in enumerate(asks): + assert ask.get("evidenceType") in valid, f"asks[{i}].evidenceType missing/invalid" + + def test_set_test_rca_error_callback_does_not_500(self): + # Error callback (success=false, no rca data) must record the error state + # and return 200 — the data-gate must be scoped to success=true. + resp = requests.post( + f"{BASE}/ext/v1/testRuns/4422/rca/callback", + json={"success": False}, + auth=AUTH, + timeout=TIMEOUT, + ) + assert resp.status_code == 200, ( + f"error callback returned {resp.status_code}; data-gate not scoped to success" + ) diff --git a/automation/upload.sh b/automation/upload.sh new file mode 100755 index 0000000..cd4f3bf --- /dev/null +++ b/automation/upload.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Upload the JUnit results to BrowserStack Observability (rengg-tfa staging), +# creating the project + build that the RCA plugin then runs against. +# +# Creds come from the environment — never commit them: +# export BSTACK_USER=... BSTACK_KEY=... +# (or `source automation/.env`, which is gitignored.) +set -euo pipefail + +UPLOAD_URL="${O11Y_UPLOAD_URL:-https://upload-observability-rengg-tfa.bsstag.com/upload}" +XML="${1:-$(dirname "$0")/build-rca-failures.xml}" +PROJECT="${PROJECT_NAME:-RCA Feature Fencing}" +BUILD="${BUILD_NAME:-VRT Build}" + +: "${BSTACK_USER:?set BSTACK_USER}" +: "${BSTACK_KEY:?set BSTACK_KEY}" + +curl -sS -X POST "$UPLOAD_URL" \ + -u "${BSTACK_USER}:${BSTACK_KEY}" \ + -F "data=@${XML}" \ + -F "projectName=${PROJECT}" \ + -F "buildName=${BUILD}" +echo diff --git a/config/rca.config.json b/config/rca.config.json index b7633bb..ea1d6ff 100644 --- a/config/rca.config.json +++ b/config/rca.config.json @@ -17,7 +17,7 @@ "product_code": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, "deploy": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, "ci": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, - "k8s": { "capability": "k8s", "discoveryHints": [] }, + "k8s": { "capability": "k8s", "discoveryHints": ["k8s-rengg-tfa"] }, "kibana": { "capability": "logs", "discoveryHints": [] }, "metrics": { "capability": "metrics", "discoveryHints": [] }, "other": { "capability": "other", "discoveryHints": [] } From 56553222dc2a34ee9ab2571b8ae075b4be4260d2 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Wed, 24 Jun 2026 12:45:43 +0530 Subject: [PATCH 15/33] fix(rca): remove command/skill name collision so the skill body loads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both commands/rca-build.md (/tfa-rca:rca-build) and skills/rca-build/SKILL.md (name: rca-build) existed in the plugin — the slash invocation resolved ambiguously and returned a bare 'loaded' stub, forcing the agent to hunt for and re-read SKILL.md. Remove the redundant command (the skill is already slash-invocable and parses build id/mode/PRs in Step 0) and tighten the skill description (393 -> ~270 chars) so it registers cleanly. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- commands/rca-build.md | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 commands/rca-build.md diff --git a/commands/rca-build.md b/commands/rca-build.md deleted file mode 100644 index 7a7a829..0000000 --- a/commands/rca-build.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -description: Run collaborative RCA over all failed tests of a BrowserStack build ---- - -# /rca-build - -Entry point for the generic RCA harness. Drives a collaborative root-cause -analysis loop over **every failed test** of a build, generic across product and -infra. - -## Input - -`$ARGUMENTS` carries the build id (and optional flags). Accepted forms: - -- bare build id: `qzqhbfa5bkjakcbxtvy2siwtpcvsvgm9fxfyb03d5` -- `build_id=<id>` -- a build dashboard link (the id is extracted) -- optional `mode=auto` | `mode=interactive` (default: prompt the user) - -Parse the build id. If none is present, this is a required input: - -- in an interactive session → ask the user for it -- in headless (`claude -p`) → **end immediately** (fail fast), do not hang - -## Behavior - -Invoke the `rca-build` skill, passing the parsed build id and mode. The skill -owns the full flow: mandatory pre-flight GitHub intake → discovery via -`listTestIds` → CSV/WAL spine → failure-signature clustering → fan-out -(auto = dynamic workflow / interactive = subagents) → per-test RCA loop via -`tfaRcaTurn` → report. - -Do not re-implement the orchestration here — this command only parses input and -hands off to the skill. From 2a14598f37eb3cbfb73d2e0e15c336bf510cf4e6 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Wed, 24 Jun 2026 13:03:30 +0530 Subject: [PATCH 16/33] =?UTF-8?q?fix(rca):=20auto=20mode=20must=20not=20bl?= =?UTF-8?q?ock=20on=20intake=20=E2=80=94=20proceed=20with=20whatever=20is?= =?UTF-8?q?=20present?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Auto mode is autonomous (no mid-run user input), but Step 1 said the intake was 'mandatory, both modes', so the agent prompted and STALLED waiting for intake answers even with mode=auto. Rewrite Step 1 per mode: auto gathers intake from invocation args + cheap inference (gh repo view, current branch), records any missing field as 'I don't have one' (RCA-only), shows a one-line FYI, and continues immediately — never prompting. Only interactive mode asks A1, once, upfront. Headless unchanged (build id required, rest default to none). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- skills/rca-build/SKILL.md | 48 ++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index 5248bee..431a521 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -1,6 +1,6 @@ --- name: rca-build -description: Run collaborative root-cause analysis over ALL failed tests of a BrowserStack build. Generic across product and infra. Mandatory pre-flight GitHub intake, then discovery via listTestIds, failure-signature clustering, and per-test RCA via tfaRcaTurn (auto = dynamic workflow / interactive = subagents). Use when a build is red and you want a per-test RCA for every failure in the TRA dashboard. +description: Batch collaborative RCA over every failed test of a BrowserStack build via tfaRcaTurn. Clusters failures, routes evidence (GitHub/k8s/logs/metrics), writes a per-test RCA. Generic across product and infra. Use when a build is red. Args: build id, optional mode=auto|interactive. --- # rca-build — batch collaborative RCA over a build @@ -20,29 +20,41 @@ Config (concurrency, turn-cap, paths, evidence registry) lives in ## Step 0 — mode + input -Parse from `/rca-build` args: the build id and optional `mode=auto|interactive`. +Parse the build id from the invocation args. Accepted forms: a bare build id, a +`build_id=<id>` token, or a build dashboard link (extract the id). Also accept an +optional `mode=auto|interactive` and any PR URLs the user supplies (carry them +into the pre-flight intake as the product/automation PRs). - No build id present → it is required: - interactive session → ask the user. - **headless (`claude -p`) with build id missing → end immediately (fail fast).** - No mode given → ask the user once (auto vs interactive). In headless, default `auto`. -## Step 1 — pre-flight intake (F1, mandatory, both modes) - -Ask the user (A1) for, in one pass: - -- product repo name, automation (test) repo name -- working branch, default branch -- the PRs in play (product + automation) -- the build id (if not already supplied) - -Every question is **mandatory to ask** but answerable with **"I don't have one"** -→ record the gap and proceed **RCA-only** (BrowserStack-side evidence + whatever -infra skills exist). Do not block the run on missing GitHub context. - -**Headless rule:** in `claude -p`, any *required* input still missing after -parsing (build id) ends the run immediately. Optional intake answers default to -"none" without prompting. +## Step 1 — pre-flight intake (F1) + +Intake fields: product repo, automation (test) repo, working branch, default +branch, the PRs in play (product + automation), and the build id. **How they're +collected depends on mode — auto must never block.** + +**Auto mode → do NOT prompt. Proceed with whatever is present.** Gather the +intake from the invocation args (build id + any PR URLs / repos / branch the user +passed) and from cheap inference (e.g. `gh repo view`, current branch). Any field +not supplied is recorded as "I don't have one" and the run proceeds **RCA-only** +for that field. Auto mode is autonomous — it does not stop to ask the user, in an +interactive `claude` session or otherwise. Show the resolved intake + capability +manifest as a one-line FYI, then immediately continue to Step 2. (This is the +"present human answered at launch by passing args" assumption — the absence of an +arg is itself the answer, not a reason to wait.) + +**Interactive mode → ask A1 once, in one pass**, for the fields above. Every +question is answerable with "I don't have one" → record the gap and proceed +RCA-only. Do not block the run on missing GitHub context. After this single +upfront pass, the rest of the batch runs without re-prompting (gaps surface via +the per-test gap-return, not the intake). + +**Headless rule:** in `claude -p`, the build id is the only required input; if +it's missing after parsing, end immediately (fail fast). All intake fields +default to "none" without prompting (same as auto). ## Step 2 — discovery (F2) From d97db23a3b9b5c0a9a1e08cd7017109b58a5adbb Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Wed, 24 Jun 2026 15:18:42 +0530 Subject: [PATCH 17/33] fix(rca): dispatch the coordinator by its plugin-namespaced agent type Plugin agents register under <plugin>:<name>, so the auto workflow's agent({agentType:'ai-tfa-coordinator'}) failed with 'agent type not found'. Use 'tfa-rca:ai-tfa-coordinator' in rca-batch.mjs (both representative + sibling dispatch) and fix the interactive-mode / agent-doc dispatch examples to match. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- agents/ai-tfa-coordinator.md | 4 ++-- skills/rca-build/references/interactive-mode.md | 3 ++- workflows/rca-batch.mjs | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 8f5cb4f..3de14aa 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -1,8 +1,8 @@ --- name: ai-tfa-coordinator description: 'Per-test collaborative-RCA coordinator. Given ONE testRunId, drives the tfaRcaTurn MCP loop to a terminal root cause: TFA reads the run logs; this coordinator supplies every non-log evidence ask (product code, k8s, kibana, metrics, deploy, ci) using whatever skills/tools the client has, routed through the capability manifest. Skips every test_logs ask (TFA owns logs). Emits a structured RCA_OUTPUT block. Generic over product and infra — no hardcoded tools. Examples: -- orchestrator: Agent(subagent_type="ai-tfa-coordinator", prompt="RCA testRunId=39 — error: empty buildName rejected on POST /builds") → drives the loop, returns RCA_OUTPUT -- sibling confirm: Agent(subagent_type="ai-tfa-coordinator", prompt="RCA testRunId=40 — pre-seed: cause=<rep root cause>, suspect PR=#7421") → one-turn confirm against this test logs +- orchestrator: Agent(subagent_type="tfa-rca:ai-tfa-coordinator", prompt="RCA testRunId=39 — error: empty buildName rejected on POST /builds") → drives the loop, returns RCA_OUTPUT +- sibling confirm: Agent(subagent_type="tfa-rca:ai-tfa-coordinator", prompt="RCA testRunId=40 — pre-seed: cause=<rep root cause>, suspect PR=#7421") → one-turn confirm against this test logs - user: "run collaborative RCA on test run 39" → single-test loop to RESOLVED/BLOCKED/PENDING' tools: [Bash, Read, Grep, Glob, Task, mcp__*__tfaRcaTurn, mcp__github__*] model: sonnet diff --git a/skills/rca-build/references/interactive-mode.md b/skills/rca-build/references/interactive-mode.md index 493ea65..0cd3042 100644 --- a/skills/rca-build/references/interactive-mode.md +++ b/skills/rca-build/references/interactive-mode.md @@ -19,7 +19,8 @@ ask-and-resume loop. ``` 1. Take the next ≤5 pending work items (representatives first, then siblings). -2. Dispatch one ai-tfa-coordinator subagent per item, mode=interactive, passing +2. Dispatch one `tfa-rca:ai-tfa-coordinator` subagent per item (plugin agents are + namespaced by plugin name — use the `tfa-rca:` prefix), mode=interactive, passing the manifest + pre-computed build evidence + (for siblings) the pre-seed. 3. Each subagent runs its loop until either: - a terminal status → returns RCA_OUTPUT (the orchestrator flips the CSV row), or diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs index e577dbe..018330d 100644 --- a/workflows/rca-batch.mjs +++ b/workflows/rca-batch.mjs @@ -107,7 +107,7 @@ const results = await pipeline( agent(siblingPrompt(sib, rca, cluster), { label: `sib:${sib.testRunId}`, phase: "Siblings", - agentType: "ai-tfa-coordinator", + agentType: "tfa-rca:ai-tfa-coordinator", schema: RCA_SCHEMA, }), ), From 1e941a35f35fd41e3b3e522282d255f052a03c10 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Wed, 24 Jun 2026 15:19:29 +0530 Subject: [PATCH 18/33] fix(rca): namespace the representative dispatch too (missed by replace_all) The prior fix only updated the sibling call (different indentation); the representative agent() at the top of the pipeline still used the bare 'ai-tfa-coordinator' and would fail identically. Both now use the namespaced type. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- workflows/rca-batch.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs index 018330d..07868e9 100644 --- a/workflows/rca-batch.mjs +++ b/workflows/rca-batch.mjs @@ -98,7 +98,7 @@ const results = await pipeline( agent(repPrompt(cluster), { label: `rep:${cluster.representative.testRunId}`, phase: "Representatives", - agentType: "ai-tfa-coordinator", + agentType: "tfa-rca:ai-tfa-coordinator", schema: RCA_SCHEMA, }).then((rca) => ({ cluster, rca })), ({ cluster, rca }) => From f07f435067fb8370c8d0aa9aacac1aedf56df7fb Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Mon, 13 Jul 2026 00:22:59 +0530 Subject: [PATCH 19/33] =?UTF-8?q?feat(rca):=20rename=20skill=20to=20/facto?= =?UTF-8?q?ry=20=E2=80=94=20single-gate,=20auto-only=20flow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - skills/rca-build/ -> skills/factory/ (frontmatter name: factory); no command file with the same name (collision lesson, see 2af22ef) - delete the mode concept: no mode arg, no auto|interactive fork, drop references/interactive-mode.md - restructure to ONE GATE before execution: Part A = connector discovery + cheap probe validation into a connector -> valid|invalid|absent manifest (gaps recorded, never blockers); Part B = intake resolved by assumption, at most ONE consolidated question at gate close (headless never asks). After gate close the run never asks the user again. - no local report: drop references/report-format.md and config paths.reportFile; finish = glimpse table + triggerRcaReport + Test Observability UI link - PRODUCT_BUG mandate in github-evidence.md: application-bug RCA must carry culprit PR link(s) or explicitly state what was searched - server emits only NEEDS_INFO|RESOLVED: drop BLOCKED from references Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- config/rca.config.json | 5 +- skills/factory/SKILL.md | 204 ++++++++++++++++++ .../references/clustering.md | 2 +- .../references/evidence-routing.md | 42 ++-- .../references/github-evidence.md | 24 ++- skills/rca-build/SKILL.md | 137 ------------ .../rca-build/references/interactive-mode.md | 57 ----- skills/rca-build/references/report-format.md | 45 ---- 8 files changed, 252 insertions(+), 264 deletions(-) create mode 100644 skills/factory/SKILL.md rename skills/{rca-build => factory}/references/clustering.md (97%) rename skills/{rca-build => factory}/references/evidence-routing.md (83%) rename skills/{rca-build => factory}/references/github-evidence.md (76%) delete mode 100644 skills/rca-build/SKILL.md delete mode 100644 skills/rca-build/references/interactive-mode.md delete mode 100644 skills/rca-build/references/report-format.md diff --git a/config/rca.config.json b/config/rca.config.json index ea1d6ff..021a72c 100644 --- a/config/rca.config.json +++ b/config/rca.config.json @@ -1,5 +1,5 @@ { - "$comment": "Central config for the generic RCA harness. All formerly-hardcoded product/infra values live here. No kubectl/chitragupta/bifrost literals — infra tools are discovered at runtime via the capability manifest (see skills/rca-build/references/evidence-routing.md).", + "$comment": "Central config for the /factory RCA harness. All formerly-hardcoded product/infra values live here. No kubectl/chitragupta/bifrost literals — connectors are discovered and probe-validated at the gate into the capability manifest (see skills/factory/references/evidence-routing.md). No reportFile: the plugin never writes a local RCA report — the full report lives on the Test Observability UI (triggerRcaReport).", "mcpServerName": "bstack", "concurrency": 5, "turnCap": 6, @@ -9,8 +9,7 @@ "errorSummaryMaxChars": 200, "paths": { "stateDir": ".rca", - "csvFile": ".rca/rca-state.csv", - "reportFile": ".rca/rca-report.md" + "csvFile": ".rca/rca-state.csv" }, "evidenceRouting": { "test_logs": { "owner": "tfa", "skip": true }, diff --git a/skills/factory/SKILL.md b/skills/factory/SKILL.md new file mode 100644 index 0000000..f2dc52e --- /dev/null +++ b/skills/factory/SKILL.md @@ -0,0 +1,204 @@ +--- +name: factory +description: Single-gate autonomous batch RCA over every failed test of a BrowserStack build via tfaRcaTurn. One gate (connector validation + assumed intake), then fully autonomous — clusters failures, routes evidence, triggers the dashboard report. Args: build id, optional PR URLs / repo hints. +--- + +# factory — single-gate autonomous RCA over a build + +Drives the `tfaRcaTurn` collaborative loop over **every failed test** of a build +and lands a per-test RCA in the TRA (Test Observability) dashboard. **TFA owns +logs; the client agent owns everything else** (product code, k8s, kibana, +metrics, deploy, ci) — routed by capability, generic over product and infra. + +This skill is the **build-level orchestrator** (`ai-tfa-orchestrator` role). It +never calls `tfaRcaTurn` itself — it dispatches the `ai-tfa-coordinator` +(test-level) per test/cluster member, which drives the loop and lets TFA author +the dashboard RCA. **The full RCA report lives on the Test Observability UI, not +in Claude** — this run's job is to feed it, then surface a terse glimpse and the +link. + +There is exactly **one mode**: autonomous. There is exactly **one gate** (Step +1) before execution. After the gate closes, **the run never asks the user +anything again.** + +Config (concurrency, turn-cap, paths, evidence registry) lives in +`config/rca.config.json`. State lives in the CSV/WAL spine (`lib/csv-state.mjs`). + +## Step 0 — input + +Parse the build id from the invocation args. Accepted forms: a bare build id, a +`build_id=<id>` token, or a build dashboard link (extract the id). Also accept +any **PR URLs** and **repo hints** (product/automation repo names or paths) the +user supplies — carry them into Gate Part B as pre-answered intake. + +- No build id present: + - interactive session → fold "which build?" into the gate's single + consolidated question (Step 1, Part B). It is the only genuinely + load-bearing field. + - **headless (`claude -p`) → end immediately (fail fast).** + +## Step 1 — THE GATE (one gate, two parts, closes once) + +Everything the run could possibly need from the user is settled here, in one +pass. The gate has two parts; both run before any RCA work starts. + +### Part A — connector discovery + validation + +Enumerate every connector relevant to test RCA: + +- from `config/rca.config.json` → `evidenceRouting`: **github** + (product_code/deploy/ci), **k8s**, **logs** (e.g. kibana), **metrics**, + **other**; +- plus any connector-shaped skills / MCP servers present in the session + (a log-search MCP, a metrics MCP, an infra skill, …). + +**Validate** each with a cheap probe — discovery alone is not enough: + +| Connector | Probe | +|---|---| +| github | `gh auth status` (or a GitHub MCP tool listed) | +| k8s | k8s skill present **and** `kubectl` reachable (e.g. `kubectl version --request-timeout=5s`) | +| logs | a log-search skill/MCP tool actually listed in the session | +| metrics | a metrics skill/MCP tool actually listed in the session | +| other | best-effort; default `absent` | + +Output the **validated capability manifest**: `connector → valid | invalid | +absent` (`lib/routing.mjs` → `buildManifest`; `valid` maps to +`available: true`). An `invalid` or `absent` connector is a **recorded gap** — +declared to the user in the gate summary and to TFA on the first turn ("I don't +have logs/metrics access") — **never a blocker**. The run always proceeds. + +### Part B — requirements (assume first, ask once, at most) + +Intake fields: product repo, automation (test) repo, working branch, default +branch, the PRs in play, and the build id. **Resolve every field by ASSUMPTION +wherever possible** — this is an assumption-OK workflow; less user interaction +is the point: + +- invocation args (build id, PR URLs, repo hints from Step 0), +- `gh repo view` / git remotes for the repos, +- the current branch for the working branch, +- cheap inference (e.g. the automation repo is the cwd if it holds the tests). + +Record each assumption in the gate summary ("assumed product repo = +`org/obs-api` from git remote"). A field that cannot be assumed is recorded as +"none" and the run proceeds RCA-only for it — **unless** it is both genuinely +non-assumable AND load-bearing (in practice: only the build id, and rarely an +ambiguous repo when PRs were supplied). Those, and only those, may be asked +**ONCE, in a single consolidated question at gate close**. Never a second +question. **Headless: skip asking entirely; record the gaps.** + +### Gate close + +Print a one-screen summary: resolved intake (with assumptions marked) + the +validated capability manifest (with gaps named). Then the gate closes. + +**AFTER THE GATE CLOSES, THE RUN NEVER ASKS THE USER ANYTHING AGAIN.** RCA +execution is fully autonomous: every downstream evidence gap becomes an +`unavailable` block back to TFA (best-effort finalize), never a prompt. + +## Step 2 — discovery + +Call the bundled MCP tool: + +``` +listTestIds(buildId=<id>, status="failed", includeFailureDetail=true) +``` + +`includeFailureDetail=true` returns each row's trimmed failure signature +(`failure.{category, error_summary, file_path, …}`) — the seed for clustering, +so no per-test probe turns are needed. + +Seed the CSV/WAL spine from the payload (`lib/csv-state.mjs` → `seed`): one row +per failed test, every row `rca_done=pending`, signature columns populated. +Re-running `seed` on an existing CSV is idempotent and preserves terminal rows +(resume-safe). If `listTestIds` returns empty → write an empty CSV, report "no +failed tests", stop. + +## Step 3 — failure-signature clustering (see references/clustering.md) + +Compute a failure signature per row and assign `cluster_id` (`lib/signature.mjs`). +Each cluster gets one **representative** (full multi-turn loop) and `N−1` +**siblings** (pre-seeded one-turn confirm against their own logs). This collapses +the expensive evidence hunt to O(distinct causes) while every test still lands a +per-test RCA. Singleton clusters are just plain per-test loops. + +## Step 4 — build-evidence pre-compute (see references/evidence-routing.md) + +Once, before fan-out (the capability manifest already exists from Gate Part A — +reuse it, do not re-discover): + +- **Build-level evidence** — compute the last-green→this-build delta (diff, + deploy timeline, suspect-PR window) **once** and pre-seed every coordinator + with the same grounded window. Cache by `(repo, commit-range)`. No "last green" + baseline (never-green suite) → fall back to a configured baseline ref and log it. + +## Step 5 — fan-out (fully autonomous) + +Drive the cluster work-list, **`concurrency` (default 5) at a time**: +representatives deep, siblings one-turn-confirm. Eagerly persist to the CSV/WAL +(claim → heartbeat → flip) so the run is resumable. + +- Claude Code → run the dynamic workflow `workflows/rca-batch.mjs` + (script-orchestrated; gap → "unavailable" back to TFA → best-effort finalize). +- Hosts without the Workflow runtime → dispatch `tfa-rca:ai-tfa-coordinator` + subagents ≤ `concurrency` at a time, or drive the sequential harness + `lib/loop.mjs` (`runRcaLoop`). Same contract, same no-prompt rule. + +Subagents/coordinators return compact `RCA_OUTPUT` blocks, never transcripts. A +coordinator that dies becomes a recorded `failed` row — one stuck test never +sinks the batch (partial-first). No path ever prompts the user (the gate is +closed). + +**Application bugs need a culprit PR.** Whenever a test's RCA classifies as +PRODUCT_BUG / application bug, the coordinator MUST hunt the culprit PR via the +github connector (deploy timeline vs last-pass window, changed paths vs failure +signature — `references/github-evidence.md`) and feed the PR link(s) to TFA in +the turn message so the dashboard RCA's `related_prs` populates. An +application-bug RCA with no GitHub PR link is **incomplete**: keep digging until +the turn cap; if still none, the turn must explicitly state "no culprit PR +identified after <what was searched>" and the CSV row records the gap. + +## Step 6 — finish: glimpse + dashboard report (NO local report) + +This plugin **never renders or writes a local RCA report**. When every row is +terminal: + +1. Print a **terse glimpse table** from the CSV (`lib/glimpse.mjs` → + `renderGlimpse`): one line per test — `testRunId → cluster → status → + confidence one-liner`. That is the entire in-Claude output. +2. Call the MCP tool **`triggerRcaReport(buildUuid=<build id>)`** (add + `force=true` only to re-run over an existing completed report). It returns a + trimmed glimpse (`state, verdict, verdictProvisional, partial, analyzedCount, + totalFailedCount, totalPrs, faultyPrNumbers, failureReason, viewReport`). +3. Print the link line, verbatim shape: + + ``` + Full report on the Test Observability UI: <viewReport> + ``` + +Humans read the real report **there**, populated by the BrowserStack agent — +not in Claude. + +## Resume + +On startup, run the reaper (`lib/csv-state.mjs` → `reaper`) to reclaim rows +stranded `in_flight` by a crashed worker (heartbeat older than +`reaperHeartbeatTtlSec`) back to `pending`, then re-point fan-out at the CSV. +Live `threadId`/`turnId` resume the prior thread; dead threads re-run from +pending. Resuming a run does **not** reopen the gate — no new questions. +(In-session only — cross-session durability is deferred.) + +## Hard rules + +- Exactly one gate. At most one consolidated question, at gate close. **After + the gate closes, never ask the user anything.** +- An invalid/absent connector is a recorded gap, never a blocker. +- Headless + missing build id → end immediately. Headless never asks. +- Never call `tfaRcaTurn` from this skill — always via the `ai-tfa-coordinator`. +- Every failed test must end terminal in the CSV — partial-first, no abort-on-one-failure. +- Never gather `test_logs` — TFA owns logs. +- Never render/write a local RCA report — glimpse table + `triggerRcaReport` + + the Test Observability UI link only. +- A PRODUCT_BUG RCA without a GitHub PR link is incomplete — dig until the turn + cap, else state what was searched and record the gap. diff --git a/skills/rca-build/references/clustering.md b/skills/factory/references/clustering.md similarity index 97% rename from skills/rca-build/references/clustering.md rename to skills/factory/references/clustering.md index 66face8..1ac5866 100644 --- a/skills/rca-build/references/clustering.md +++ b/skills/factory/references/clustering.md @@ -47,7 +47,7 @@ Distinct failures can share an error string. A sibling's pre-seed turn is a *hypothesis to confirm*, not a verdict to copy: - TFA `RESOLVED`s the sibling in one turn → logs-grounded inheritance, cheap. -- TFA returns `NEEDS_INFO` / `BLOCKED` (the hypothesis does not hold for this +- TFA returns `NEEDS_INFO` (the hypothesis does not hold for this test's logs) → the sibling **falls back to its own full loop**. The representative's cause is never stamped onto a sibling without log confirmation. diff --git a/skills/rca-build/references/evidence-routing.md b/skills/factory/references/evidence-routing.md similarity index 83% rename from skills/rca-build/references/evidence-routing.md rename to skills/factory/references/evidence-routing.md index 87fb255..afe71aa 100644 --- a/skills/rca-build/references/evidence-routing.md +++ b/skills/factory/references/evidence-routing.md @@ -8,8 +8,9 @@ submits on the next turn. The core contract: **TFA owns logs; the client agent owns everything else.** The coordinator never seeds logs and never fulfills a `test_logs` ask. Every other `evidenceType` routes to a capability that is gathered via **whatever skill/tool -the client actually has** for it (discovered once into the capability manifest — -see `SKILL.md` § Pre-compute). There are **no `kubectl` / `chitragupta` / +the client actually has** for it (discovered **and validated** once into the +capability manifest — see `SKILL.md` § Gate Part A). There are **no `kubectl` / +`chitragupta` / `bifrost` literals here** — that is the whole point of going generic. The registry logic lives in `lib/routing.mjs` (`routeAsk` / `routeAsks`); this @@ -27,11 +28,9 @@ priority }`. For each ask, in descending `priority` (`high` → `medium` → `lo - **skip** — `test_logs` (TFA-owned). Gather nothing; record in `asks_skipped`. - **gather** — a capability is available. Run its discovered skill/tool scoped by `what` / `why`, then digest the result into one ask block. - - **gap** — no capability is available. Hand the ask to the injected - **`resolveGap()`** policy: - - **auto mode** → emit an `unavailable` block back to TFA (no user prompt). - - **interactive mode** → return the gap to the main agent, which asks the - user, then feeds the answer back. + - **gap** — no valid connector for that `evidenceType` (the gate recorded it + as `invalid`/`absent`). Emit an `unavailable` block back to TFA — **never + prompt the user** (the gate is closed; the run is autonomous). 2. Concatenate the per-ask blocks into the next-turn `message` and resubmit on the same `threadId`. @@ -125,30 +124,32 @@ SUMMARY: not-found | unreachable | unavailable | out-of-scope — <one line: wha - `not-found` — the skill/tool ran but the signal isn't there. State the search performed. - `unreachable` — the surface was not reachable from this agent context. State which. -- `unavailable` — no capability/skill exists for this `evidenceType` (auto-mode gap result). +- `unavailable` — no valid connector exists for this `evidenceType` (a gate-recorded gap). - `out-of-scope` — the ask is `test_logs` or otherwise not the agent's to fulfill. -An all-`unavailable` / all-`not-found` turn still resubmits — TFA decides whether -the gap is fatal (→ BLOCKED) or it can converge anyway (best-effort, lower -confidence). The coordinator does not pre-empt that decision. +An all-`unavailable` / all-`not-found` turn still resubmits — TFA decides how to +converge (best-effort, lower confidence) or what else to ask. The coordinator +does not pre-empt that decision. --- -## Capability manifest (built once per run) +## Capability manifest (built once, at the gate) Rather than re-discover "is there a kibana skill?" on every ask across every -test, the orchestrator enumerates the client's available skills/tools **once** up -front into a manifest (`lib/routing.mjs` → `buildManifest`): +test, Gate Part A enumerates **and probe-validates** the client's connectors +**once** up front into a manifest (`lib/routing.mjs` → `buildManifest`). +`valid` maps to `available: true`; `invalid`/`absent` map to `available: false` +(a recorded gap): ``` -{ github: {available: true, via: "github-mcp"}, k8s: {available: false}, ... } +{ github: {available: true, via: "gh"}, k8s: {available: false}, ... } ``` - Every ask routes against this manifest — reproducible, no per-ask discovery. -- The orchestrator **declares the unavailable capabilities to the user** up front - ("k8s + metrics will be unavailable") and includes them in the first turn so - TFA plans asks around what's obtainable. -- Frozen at run start. A skill appearing mid-run is not picked up until the next run. +- The gate summary **declares the gaps to the user** ("k8s + metrics not + available") and the first turn declares them to TFA so it plans asks around + what's obtainable. +- Frozen at gate close. A skill appearing mid-run is not picked up until the next run. ## Build-level evidence cache (compute once) @@ -159,4 +160,5 @@ last-green→this-build delta **once** (`lib/evidence-cache.mjs`), caches it by same grounded suspect window — collapsing N×M redundant git/infra calls to ~M and front-loading the highest-signal evidence so many tests RESOLVE before any infra ask fires. No "last green" (never-green suite) → fall back to a configured -baseline ref and note the weaker grounding in the report. +baseline ref and note the weaker grounding in the turn digest (it lands in the +dashboard RCA). diff --git a/skills/rca-build/references/github-evidence.md b/skills/factory/references/github-evidence.md similarity index 76% rename from skills/rca-build/references/github-evidence.md rename to skills/factory/references/github-evidence.md index fa24aaa..ae1ddcc 100644 --- a/skills/rca-build/references/github-evidence.md +++ b/skills/factory/references/github-evidence.md @@ -16,9 +16,31 @@ that tries to *disprove* each suspect before it enters `related_prs`. `gh api`, `merge-base`, ancestry) and anything the MCP doesn't cover. 3. **Neither** → emit an `unavailable` block for the ask (do not fabricate a PR). -The orchestrator records which is present in the capability manifest +The gate records which is present **and probe-validated** (`gh auth status` / +GitHub MCP tools listed) in the capability manifest (`capability: github → { available, via }`); route every github ask against it. +## Application bugs REQUIRE a culprit-PR hunt (mandatory) + +Whenever TFA's working classification is **PRODUCT_BUG / application bug**, the +github connector is not optional evidence — it is the deliverable. The +coordinator MUST hunt the culprit PR: + +1. **Deploy timeline vs last-pass window** — what shipped to the run's env + between the last passing run and this failure. +2. **Changed paths vs failure signature** — intersect the window's PRs' changed + files with the failing file/function from the signature. +3. Run the falsification protocol below on each candidate. + +Feed the surviving PR **link(s)** to TFA in the turn message so the BrowserStack +agent populates `related_prs` in the dashboard RCA. **An application-bug RCA +with no GitHub PR link is INCOMPLETE**: keep digging on subsequent turns until +the turn cap. If still none, the turn must explicitly state +`no culprit PR identified after <what was searched: window, repos, paths>` and +the CSV row records the gap. Never fabricate a PR; if the github connector is +invalid/absent, the same explicit statement plus an `unavailable` block goes to +TFA (a gate-recorded gap). + ## Evidence each ask needs (be specific — no fishing) | Ask intent | Gather exactly | diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md deleted file mode 100644 index 431a521..0000000 --- a/skills/rca-build/SKILL.md +++ /dev/null @@ -1,137 +0,0 @@ ---- -name: rca-build -description: Batch collaborative RCA over every failed test of a BrowserStack build via tfaRcaTurn. Clusters failures, routes evidence (GitHub/k8s/logs/metrics), writes a per-test RCA. Generic across product and infra. Use when a build is red. Args: build id, optional mode=auto|interactive. ---- - -# rca-build — batch collaborative RCA over a build - -Drives the `tfaRcaTurn` collaborative loop over **every failed test** of a build -and records a per-test RCA. **TFA owns logs; the client agent owns everything -else** (product code, k8s, kibana, metrics, deploy, ci) — routed by capability, -generic over product and infra. - -This skill is the **build-level orchestrator** (`ai-tfa-orchestrator` role). It -never calls `tfaRcaTurn` itself — it dispatches the `ai-tfa-coordinator` -(test-level) per test/cluster member, which drives the loop and lets TFA author -the dashboard RCA. - -Config (concurrency, turn-cap, paths, evidence registry) lives in -`config/rca.config.json`. State lives in the CSV/WAL spine (`lib/csv-state.mjs`). - -## Step 0 — mode + input - -Parse the build id from the invocation args. Accepted forms: a bare build id, a -`build_id=<id>` token, or a build dashboard link (extract the id). Also accept an -optional `mode=auto|interactive` and any PR URLs the user supplies (carry them -into the pre-flight intake as the product/automation PRs). - -- No build id present → it is required: - - interactive session → ask the user. - - **headless (`claude -p`) with build id missing → end immediately (fail fast).** -- No mode given → ask the user once (auto vs interactive). In headless, default `auto`. - -## Step 1 — pre-flight intake (F1) - -Intake fields: product repo, automation (test) repo, working branch, default -branch, the PRs in play (product + automation), and the build id. **How they're -collected depends on mode — auto must never block.** - -**Auto mode → do NOT prompt. Proceed with whatever is present.** Gather the -intake from the invocation args (build id + any PR URLs / repos / branch the user -passed) and from cheap inference (e.g. `gh repo view`, current branch). Any field -not supplied is recorded as "I don't have one" and the run proceeds **RCA-only** -for that field. Auto mode is autonomous — it does not stop to ask the user, in an -interactive `claude` session or otherwise. Show the resolved intake + capability -manifest as a one-line FYI, then immediately continue to Step 2. (This is the -"present human answered at launch by passing args" assumption — the absence of an -arg is itself the answer, not a reason to wait.) - -**Interactive mode → ask A1 once, in one pass**, for the fields above. Every -question is answerable with "I don't have one" → record the gap and proceed -RCA-only. Do not block the run on missing GitHub context. After this single -upfront pass, the rest of the batch runs without re-prompting (gaps surface via -the per-test gap-return, not the intake). - -**Headless rule:** in `claude -p`, the build id is the only required input; if -it's missing after parsing, end immediately (fail fast). All intake fields -default to "none" without prompting (same as auto). - -## Step 2 — discovery (F2) - -Call the bundled MCP tool: - -``` -listTestIds(buildId=<id>, status="failed", includeFailureDetail=true) -``` - -`includeFailureDetail=true` returns each row's trimmed failure signature -(`failure.{category, error_summary, file_path, …}`) — the seed for clustering, so -no per-test probe turns are needed. - -Seed the CSV/WAL spine from the payload (`lib/csv-state.mjs` → `seed`): one row -per failed test, every row `rca_done=pending`, signature columns populated. -Re-running `seed` on an existing CSV is idempotent and preserves terminal rows -(resume-safe). If `listTestIds` returns empty → write an empty CSV, report "no -failed tests", stop. - -## Step 3 — failure-signature clustering (see references/clustering.md) - -Compute a failure signature per row and assign `cluster_id` (`lib/signature.mjs`). -Each cluster gets one **representative** (full multi-turn loop) and `N−1` -**siblings** (pre-seeded one-turn confirm against their own logs). This collapses -the expensive evidence hunt to O(distinct causes) while every test still lands a -per-test RCA. Singleton clusters are just plain per-test loops. - -## Step 4 — build-evidence pre-compute + capability manifest (see references/evidence-routing.md) - -Once, before fan-out: - -- **Capability manifest** — enumerate the skills/tools the client actually has - into `capability → {available, via}` (GitHub, k8s, logs, metrics, …). Declare - to the user up front what will be **unavailable** ("k8s + metrics not - available"). Every coordinator routes asks against this manifest. -- **Build-level evidence** — compute the last-green→this-build delta (diff, - deploy timeline, suspect-PR window) **once** and pre-seed every coordinator - with the same grounded window. Cache by `(repo, commit-range)`. No "last green" - baseline (never-green suite) → fall back to a configured baseline ref and log it. - -## Step 5 — fan-out (the mode fork) - -Drive the cluster work-list, **`concurrency` (default 5) at a time**: -representatives deep, siblings one-turn-confirm. Eagerly persist to the CSV/WAL -(claim → heartbeat → flip) so the run is resumable. - -- **auto** → run the dynamic workflow `workflows/rca-batch.mjs` (script-orchestrated, - no user input; gap → "unavailable" back to TFA → best-effort finalize). -- **interactive** → spawn `ai-tfa-coordinator` subagents 5 at a time; on an - evidence gap a subagent ends early with a `GAP_OUTPUT` (resume handles), and - this orchestrator asks the user (A1) then re-dispatches with `resume=`. Subagents - return compact blocks, not transcripts (keeps the main context lean for large - batches). Full protocol: `references/interactive-mode.md`. - -Both modes use the **same** `ai-tfa-coordinator`; only the injected gap-resolver -differs. A coordinator that dies becomes a recorded `failed` row — one stuck test -never sinks the batch (partial-first). - -## Step 6 — report (see references/report-format.md) - -When every row is terminal, render the report (`paths.reportFile`): per-test rows -with status + the **evidence-coverage band** (a RESOLVED built with evidence -unavailable reads as lower confidence than a fully-evidenced one). Degrade, -don't crash — missing fields render as "not available". - -## Resume - -On startup, run the reaper (`lib/csv-state.mjs` → `reaper`) to reclaim rows -stranded `in_flight` by a crashed worker (heartbeat older than -`reaperHeartbeatTtlSec`) back to `pending`, then re-point fan-out at the CSV. -Live `threadId`/`turnId` resume the prior thread; dead threads re-run from -pending. (In-session only — cross-session durability is deferred.) - -## Hard rules - -- Always run the pre-flight intake; never silently skip it (but never block on "I don't have one"). -- Headless + missing required input → end immediately. -- Never call `tfaRcaTurn` from this skill — always via the `ai-tfa-coordinator`. -- Every failed test must end terminal in the CSV — partial-first, no abort-on-one-failure. -- Never gather `test_logs` — TFA owns logs. diff --git a/skills/rca-build/references/interactive-mode.md b/skills/rca-build/references/interactive-mode.md deleted file mode 100644 index 0cd3042..0000000 --- a/skills/rca-build/references/interactive-mode.md +++ /dev/null @@ -1,57 +0,0 @@ -# Interactive mode — subagents with a user in the loop - -Interactive mode (D2) puts the human (A1) in the loop **only at the orchestrator -layer**. The main session spawns `ai-tfa-coordinator` subagents to investigate in -parallel; when a subagent needs evidence it can't get, it hands the gap back up to -the orchestrator, which asks the user and feeds the answer down. - -This is the **same coordinator** the auto workflow uses — only the gap-resolver -differs (auto → "unavailable"; interactive → return the gap). - -## Why a subagent can't just "ask the user" - -A dispatched subagent runs to completion and returns one final message — it -cannot pause mid-run, prompt the user, and resume. So the gap-return is modeled -as **early termination with resume handles**, and the orchestrator drives the -ask-and-resume loop. - -## The orchestrator loop (per batch of ≤ `concurrency`, default 5) - -``` -1. Take the next ≤5 pending work items (representatives first, then siblings). -2. Dispatch one `tfa-rca:ai-tfa-coordinator` subagent per item (plugin agents are - namespaced by plugin name — use the `tfa-rca:` prefix), mode=interactive, passing - the manifest + pre-computed build evidence + (for siblings) the pre-seed. -3. Each subagent runs its loop until either: - - a terminal status → returns RCA_OUTPUT (the orchestrator flips the CSV row), or - - an interactive GAP → returns GAP_OUTPUT (status=PENDING) carrying: - { testRunId, threadId, turnId, gap: { evidenceType, what, why } } -4. For each GAP_OUTPUT: ASK A1 for that evidence (one focused question). - - A1 answers → re-dispatch a coordinator with resume={threadId,turnId} and - the answer digested into the next turn's message. Continue its loop. - - A1 has nothing → tell the coordinator to report "unavailable" on resume - (degrade exactly like auto for that one ask). -5. Repeat until every row is terminal. Then dispatch the next batch. -``` - -## Aggregation discipline (large batches) - -Subagents return **compact `RCA_OUTPUT` / `GAP_OUTPUT` blocks, never transcripts** -— mirroring the auto workflow's "results in script vars" rule — so the main -agent's context stays lean even over hundreds of tests. The orchestrator never -holds full per-test loop transcripts; it holds one block per test. - -## Partial-first - -A subagent that dies becomes a recorded `failed` row (the orchestrator -synthesizes it). One stuck test never sinks the batch — same contract as auto. - -## When to prefer interactive over auto - -- The client is missing infra skills the failures clearly need (k8s/kibana), and - the user can supply that evidence by hand. -- The user wants to steer or sign off mid-run. - -Otherwise auto is cheaper (no human round-trips). Both write the same CSV rows -and the same report, so a run can start auto and the residual BLOCKED/gap tests -can be re-run interactively (the auto-first / escalate-the-residue pattern). diff --git a/skills/rca-build/references/report-format.md b/skills/rca-build/references/report-format.md deleted file mode 100644 index e27a28d..0000000 --- a/skills/rca-build/references/report-format.md +++ /dev/null @@ -1,45 +0,0 @@ -# Report format, coverage stamp, and resume - -## The CSV is the source of truth - -Every per-test result lives as one CSV row (`lib/csv-state.mjs`, columns in -`COLUMNS`). The report is a deterministic render of that CSV — no per-test -transcripts are kept. `rca_done` ∈ `pending | resolved | blocked | failed | -pending-resume`. - -## Coverage stamp (ideation #6, v1) - -At flip time the orchestrator stamps each row (`lib/coverage.mjs`) from the -coordinator's `asks_fulfilled` / `asks_unavailable` + TFA's confidence: - -- **coverage** — `full` (no gaps) · `partial` (some fulfilled, some unavailable) · - `thin` (nothing fulfilled, only gaps). -- **band** — TFA's confidence **capped by coverage**: `full` keeps it, `partial` - caps at `medium`, `thin` caps at `low`; unknown floors to `low`. - -So a RESOLVED with kibana/k8s unavailable reads as a lower band *because* evidence -was missing — not the same as a fully-evidenced RESOLVED. The report's **Coverage -caveats** section spells this out per affected row. - -> Out of v1 scope: the build-level **blast-radius digest** (rows inverted by -> culprit PR, ranked) — deferred to follow-up. The per-row coverage stamp ships now. - -## Report layout (`lib/report.mjs` → `renderReport`) - -- Header + build id + generated-at. -- One-line summary: total + counts by `rca_done`. -- A per-test table: `testRunId | test | status | confidence | coverage | root cause | related PRs`. -- A **Coverage caveats** list for `partial`/`thin` rows. - -**Degrade, don't crash:** any missing field renders as `not available`; an empty -batch renders "No failed tests analyzed."; pipes are escaped and newlines -collapsed so the table never breaks. - -## Resume (ideation #7) - -On startup the orchestrator runs the **reaper** (`lib/csv-state.mjs` → `reaper`): -rows stuck `in_flight` with a heartbeat older than `reaperHeartbeatTtlSec` are -reclaimed to `pending` (a crashed worker's rows), then fan-out re-points at the -CSV. A row that retains a live `threadId`/`turnId` resumes that TFA thread; a dead -thread re-runs from `pending`. In-session / in-workspace only — cross-session -durability is deferred. From 7906844c53b472ab677cba0a221c42d7822893c2 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Mon, 13 Jul 2026 00:26:26 +0530 Subject: [PATCH 20/33] feat(rca): consume trimmed tfaRcaTurn shapes; glimpse output replaces local report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - lib/loop.mjs: RESOLVED now carries {glimpse:{root_cause,failure_type, related_prs}, viewRca} — no full rca payload; drop BLOCKED (server emits only NEEDS_INFO|RESOLVED, plus soft-PENDING); drop the resolveGap injection — gaps always degrade to an 'unavailable' block (auto-only, no prompt path) - delete lib/report.mjs and .rca/rca-report*.md — the plugin never renders a local RCA report; new lib/glimpse.mjs renders the terse end-of-run glimpse table (testRunId → cluster → status → confidence one-liner) from the CSV spine - fixtures: resolved.json/pending.json to the trimmed shapes; drop blocked.json; conformance + coverage tests updated (49 green) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- lib/glimpse.mjs | 52 ++++++++++++++ lib/loop.mjs | 50 +++++++------- lib/report.mjs | 69 ------------------- tests/conformance.test.mjs | 44 +++++------- ...ort.test.mjs => coverage-glimpse.test.mjs} | 59 +++++++--------- tests/fixtures/recorded-turns/blocked.json | 13 ---- tests/fixtures/recorded-turns/pending.json | 3 +- tests/fixtures/recorded-turns/resolved.json | 8 +-- 8 files changed, 124 insertions(+), 174 deletions(-) create mode 100644 lib/glimpse.mjs delete mode 100644 lib/report.mjs rename tests/{coverage-report.test.mjs => coverage-glimpse.test.mjs} (56%) delete mode 100644 tests/fixtures/recorded-turns/blocked.json diff --git a/lib/glimpse.mjs b/lib/glimpse.mjs new file mode 100644 index 0000000..63e328d --- /dev/null +++ b/lib/glimpse.mjs @@ -0,0 +1,52 @@ +// Terse end-of-run glimpse — the ONLY in-client output of a /factory run. +// One line per test: testRunId → cluster → status → confidence one-liner. +// The full RCA report lives on the Test Observability UI (triggerRcaReport → +// viewReport link); this is deliberately a glimpse, never a report. Degrade, +// don't crash: missing fields render as "-". + +import { readRows } from "./csv-state.mjs"; + +const DASH = "-"; +const ONE_LINER_MAX = 80; + +function cell(value) { + const s = value == null ? "" : String(value).trim(); + if (s === "") return DASH; + return s.replace(/\s*\n\s*/g, " "); +} + +function oneLiner(row) { + const confidence = cell(row.confidence); + const cause = cell(row.root_cause); + const text = cause === DASH ? confidence : `${confidence}: ${cause}`; + return text.length > ONE_LINER_MAX ? `${text.slice(0, ONE_LINER_MAX - 1)}…` : text; +} + +// Render the glimpse from CSV rows. Returns a plain-text block: +// <testRunId> → <cluster_id> → <status> → <confidence one-liner> +export function renderGlimpse(rows, { buildId } = {}) { + const lines = [`Glimpse${buildId ? ` — build ${buildId}` : ""}`]; + if (!rows || rows.length === 0) { + lines.push("No failed tests analyzed."); + return lines.join("\n") + "\n"; + } + const byState = rows.reduce((acc, r) => { + const k = r.rca_done || "unknown"; + acc[k] = (acc[k] ?? 0) + 1; + return acc; + }, {}); + const summary = Object.entries(byState) + .map(([k, v]) => `${k}: ${v}`) + .join(" · "); + lines.push(`${rows.length} test(s) — ${summary}`); + for (const r of rows) { + lines.push( + `${cell(r.testRunId)} → ${cell(r.cluster_id)} → ${cell(r.rca_done)} → ${oneLiner(r)}`, + ); + } + return lines.join("\n") + "\n"; +} + +export function renderGlimpseFromCsv(csvPath, opts = {}) { + return renderGlimpse(readRows(csvPath), opts); +} diff --git a/lib/loop.mjs b/lib/loop.mjs index 9e59eb2..b03ce61 100644 --- a/lib/loop.mjs +++ b/lib/loop.mjs @@ -1,12 +1,23 @@ // Executable mirror of the ai-tfa-coordinator loop (agents/ai-tfa-coordinator.md). // It drives the collaborative loop against an injected `submit` (real = the // tfaRcaTurn MCP tool; tests = a recorded-turn replayer), so the loop mechanics — -// status branching, ask routing, gap resolution, turn-cap, one-thread, +// status branching, ask routing, gap degradation, turn-cap, one-thread, // soft-PENDING — are tested rather than assumed. // -// Double duty: this is ALSO the **sequential thin-client harness** (D5 / ideation -// #4) — the third caller of the same contract, for MCP clients without -// workflows/subagents. Pure + dependency-light (imports only the routing registry). +// Double duty: this is ALSO the **sequential thin-client harness** — the third +// caller of the same contract, for MCP clients without workflows/subagents. +// Pure + dependency-light (imports only the routing registry). +// +// The loop is fully autonomous: an evidence gap ALWAYS degrades to an +// `unavailable` block back to TFA. There is no user-prompt path — the /factory +// gate closed before this loop ever runs. +// +// tfaRcaTurn returns TRIMMED terminal shapes: +// RESOLVED → { status, confidence, threadId, +// glimpse: { root_cause (≤220 chars), failure_type, related_prs }, +// viewRca } +// PENDING → { status, turnId, threadId } (soft-pending, resumable) +// NEEDS_INFO → { status, questions/asks/suggestions } (verbatim — the loop needs them) import { routeAsks } from "./routing.mjs"; @@ -16,7 +27,7 @@ function unavailableBlock(gap) { `ASK: ${what}`, `TYPE: ${gap.evidenceType}`, `FOUND: no`, - `SUMMARY: unavailable — no ${gap.capability} capability for this client.`, + `SUMMARY: unavailable — no ${gap.capability} connector for this client.`, ].join("\n"); } @@ -24,7 +35,6 @@ function unavailableBlock(gap) { // // submit({ testRunId, message, threadId, turnId }) → Promise<turn> (tfaRcaTurn shape) // gather(routedGatherEntry) → Promise<string> (one digest block) -// resolveGap(routedGapEntry) → Promise<{ digest } | null> (auto: null; interactive: a digest) export async function runRcaLoop({ testRunId, firstMessage = "", @@ -32,7 +42,6 @@ export async function runRcaLoop({ config = {}, manifest = {}, gather = async () => "", - resolveGap = async () => null, turnCap = config?.turnCap ?? 6, }) { if (testRunId == null || Number.isNaN(Number(testRunId))) { @@ -56,19 +65,15 @@ export async function runRcaLoop({ const unavailable = new Set(); const out = (status, turn, note) => { - const rca = turn?.rca ?? {}; + const glimpse = turn?.glimpse ?? {}; return { testRunId: String(testRunId), status, confidence: turn?.confidence ?? "unknown", - root_cause: - status === "RESOLVED" - ? (rca.root_cause ?? "") - : status === "BLOCKED" - ? (turn?.reason ?? "") - : (note ?? ""), - possible_fix: rca.possible_fix ?? "", - related_prs: rca.related_prs ?? [], + root_cause: status === "RESOLVED" ? (glimpse.root_cause ?? "") : (note ?? ""), + failure_type: glimpse.failure_type ?? "", + related_prs: glimpse.related_prs ?? [], + view_rca: turn?.viewRca ?? "", threadId: threadId ?? null, turnId: turnId ?? null, turns_used: turns, @@ -84,7 +89,6 @@ export async function runRcaLoop({ threadId = turn.threadId ?? threadId; if (turn.status === "RESOLVED") return out("RESOLVED", turn); - if (turn.status === "BLOCKED") return out("BLOCKED", turn); if (turn.status === "PENDING") { turnId = turn.turnId ?? turnId; return out("PENDING", turn, "soft-pending"); @@ -95,7 +99,7 @@ export async function runRcaLoop({ // would run for nothing). if (turns >= turnCap) return out("PENDING", turn, "turn-cap"); - // Route + fulfill. + // Route + fulfill. Gaps degrade to `unavailable` — never a user prompt. const buckets = routeAsks(turn.asks ?? [], config, manifest); const blocks = []; for (const s of buckets.skip) skipped.add(s.evidenceType); @@ -104,14 +108,8 @@ export async function runRcaLoop({ fulfilled.add(g.evidenceType); } for (const gap of buckets.gap) { - const resolved = await resolveGap(gap); - if (resolved && resolved.digest) { - blocks.push(resolved.digest); - fulfilled.add(gap.evidenceType); - } else { - unavailable.add(gap.evidenceType); - blocks.push(unavailableBlock(gap)); - } + unavailable.add(gap.evidenceType); + blocks.push(unavailableBlock(gap)); } message = blocks.join("\n\n"); diff --git a/lib/report.mjs b/lib/report.mjs deleted file mode 100644 index 84ae275..0000000 --- a/lib/report.mjs +++ /dev/null @@ -1,69 +0,0 @@ -// Deterministic markdown report for a finished (or partial) batch. Degrade, -// don't crash: any missing field renders as "not available"; an empty batch -// still renders a valid report. Reads the CSV/WAL spine; no per-test transcripts. - -import { readRows } from "./csv-state.mjs"; - -const NA = "not available"; - -function cell(value) { - const s = value == null ? "" : String(value).trim(); - if (s === "") return NA; - // keep the table one-line-per-row: collapse newlines, escape pipes - return s.replace(/\s*\n\s*/g, " ").replace(/\|/g, "\\|"); -} - -function countBy(rows, key) { - return rows.reduce((acc, r) => { - const k = r[key] || "unknown"; - acc[k] = (acc[k] ?? 0) + 1; - return acc; - }, {}); -} - -// Render from a rows array (testable) — or pass a csvPath via renderReportFromCsv. -export function renderReport(rows, { buildId, generatedAt } = {}) { - const lines = []; - lines.push(`# RCA report${buildId ? ` — build ${buildId}` : ""}`); - if (generatedAt) lines.push(`\nGenerated: ${generatedAt}`); - - if (!rows || rows.length === 0) { - lines.push("\nNo failed tests analyzed."); - return lines.join("\n") + "\n"; - } - - const byState = countBy(rows, "rca_done"); - const summary = Object.entries(byState) - .map(([k, v]) => `${k}: ${v}`) - .join(" · "); - lines.push(`\n**${rows.length} test(s)** — ${summary}\n`); - - lines.push( - "| testRunId | test | status | confidence | coverage | root cause | related PRs |", - ); - lines.push("|---|---|---|---|---|---|---|"); - for (const r of rows) { - lines.push( - `| ${cell(r.testRunId)} | ${cell(r.testName)} | ${cell(r.rca_done)} | ${cell( - r.confidence, - )} | ${cell(r.coverage)} | ${cell(r.root_cause)} | ${cell(r.related_prs)} |`, - ); - } - - // Surface coverage caveats so a "low confidence" reads as "because X unavailable". - const thin = rows.filter((r) => r.coverage === "thin" || r.coverage === "partial"); - if (thin.length > 0) { - lines.push(`\n## Coverage caveats`); - for (const r of thin) { - lines.push( - `- ${cell(r.testRunId)} (${cell(r.coverage)} coverage): confidence band reflects evidence that was unavailable, not just model certainty.`, - ); - } - } - - return lines.join("\n") + "\n"; -} - -export function renderReportFromCsv(csvPath, opts = {}) { - return renderReport(readRows(csvPath), opts); -} diff --git a/tests/conformance.test.mjs b/tests/conformance.test.mjs index 5ea6192..82f7b8c 100644 --- a/tests/conformance.test.mjs +++ b/tests/conformance.test.mjs @@ -22,7 +22,7 @@ const GITHUB_AVAILABLE = { github: { available: true, via: "gh" } }; // A coordinator gather() stub: returns a one-line digest block. const gather = async (g) => `ASK: ${g.ask.what}\nTYPE: ${g.evidenceType}\nFOUND: yes\nSUMMARY: stub`; -test("resolved fixture: NEEDS_INFO → evidence → RESOLVED, rca captured, test_logs skipped", async () => { +test("resolved fixture: NEEDS_INFO → evidence → RESOLVED, trimmed glimpse captured, test_logs skipped", async () => { const fx = load("resolved.json"); const result = await runRcaLoop({ testRunId: fx.testRunId, @@ -34,25 +34,17 @@ test("resolved fixture: NEEDS_INFO → evidence → RESOLVED, rca captured, test }); assert.equal(result.status, "RESOLVED"); assert.match(result.root_cause, /#7421/); + assert.ok(result.root_cause.length <= 220); // glimpse root_cause is trimmed server-side + assert.equal(result.failure_type, "product_regression"); assert.deepEqual(result.related_prs, ["#7421"]); + assert.match(result.view_rca, /^https:\/\/observability\.browserstack\.com\/builds\//); assert.deepEqual(result.asks_fulfilled, ["product_code"]); assert.deepEqual(result.asks_skipped, ["test_logs"]); // TFA-owned, never gathered assert.equal(result.turns_used, 2); assert.equal(result.threadId, "thr-39"); }); -test("blocked fixture: terminal with reason captured", async () => { - const fx = load("blocked.json"); - const result = await runRcaLoop({ - testRunId: fx.testRunId, - submit: replaySubmit(fx.turns), - config: CONFIG, - }); - assert.equal(result.status, "BLOCKED"); - assert.match(result.root_cause, /could not obtain server-side logs/); -}); - -test("pending fixture: soft-PENDING ends with turnId, no re-poll", async () => { +test("pending fixture: soft-PENDING (trimmed: status/threadId/turnId) ends with turnId, no re-poll", async () => { const fx = load("pending.json"); let calls = 0; const counting = async (args) => { @@ -66,6 +58,7 @@ test("pending fixture: soft-PENDING ends with turnId, no re-poll", async () => { }); assert.equal(result.status, "PENDING"); assert.equal(result.turnId, "turn-81-1"); + assert.equal(result.threadId, "thr-81"); assert.equal(calls, 1); // ended immediately, did not poll again }); @@ -88,34 +81,35 @@ test("turn-cap fixture: ends PENDING(turn-cap) at the cap, never a 7th submit", assert.equal(submits, 6); // capped at turnCap, never 7 }); -test("degraded path: no capability + auto resolveGap → asks_unavailable, still terminal", async () => { - // Same resolved fixture, but the client has NO github capability and runs auto - // (resolveGap returns null → 'unavailable'). The loop must still reach RESOLVED. +test("degraded path: no connector → gap degrades to unavailable (never a prompt), still terminal", async () => { + // Same resolved fixture, but the client has NO github connector. The loop is + // autonomous: the gap becomes an `unavailable` block and it still RESOLVEs. const fx = load("resolved.json"); const result = await runRcaLoop({ testRunId: fx.testRunId, submit: replaySubmit(fx.turns), config: CONFIG, - manifest: {}, // nothing available - resolveGap: async () => null, // auto: report unavailable + manifest: {}, // nothing valid at the gate }); assert.equal(result.status, "RESOLVED"); assert.deepEqual(result.asks_unavailable, ["product_code"]); assert.deepEqual(result.asks_fulfilled, []); }); -test("interactive resolveGap supplies the missing evidence → fulfilled, not unavailable", async () => { +test("unavailable block names the missing connector in the resubmitted message", async () => { const fx = load("resolved.json"); - const result = await runRcaLoop({ + const messages = []; + const recording = (inner) => async (args) => { + messages.push(args.message); + return inner(args); + }; + await runRcaLoop({ testRunId: fx.testRunId, - submit: replaySubmit(fx.turns), + submit: recording(replaySubmit(fx.turns)), config: CONFIG, manifest: {}, - resolveGap: async () => ({ digest: "ASK: ...\nFOUND: yes\nSUMMARY: user supplied" }), }); - assert.equal(result.status, "RESOLVED"); - assert.deepEqual(result.asks_fulfilled, ["product_code"]); - assert.deepEqual(result.asks_unavailable, []); + assert.match(messages[1], /unavailable — no github connector/); }); test("no testRunId → failed block, tool never called", async () => { diff --git a/tests/coverage-report.test.mjs b/tests/coverage-glimpse.test.mjs similarity index 56% rename from tests/coverage-report.test.mjs rename to tests/coverage-glimpse.test.mjs index 7d7cb94..18f2817 100644 --- a/tests/coverage-report.test.mjs +++ b/tests/coverage-glimpse.test.mjs @@ -1,7 +1,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { coverageStamp, classifyCoverage } from "../lib/coverage.mjs"; -import { renderReport } from "../lib/report.mjs"; +import { renderGlimpse } from "../lib/glimpse.mjs"; // ---- coverage stamp -------------------------------------------------------- @@ -47,62 +47,51 @@ test("classifyCoverage dedupes and handles empties", () => { assert.equal(classifyCoverage([], ["x"]), "thin"); }); -// ---- report ---------------------------------------------------------------- +// ---- glimpse (the ONLY in-client output — no local report) ------------------ -test("empty batch renders a valid report, no crash", () => { - const md = renderReport([], { buildId: "b1" }); - assert.match(md, /No failed tests analyzed/); +test("empty batch renders a valid glimpse, no crash", () => { + const txt = renderGlimpse([], { buildId: "b1" }); + assert.match(txt, /No failed tests analyzed/); }); -test("report renders a row table with status counts", () => { +test("glimpse renders one arrow line per test with status counts", () => { const rows = [ { testRunId: "101", - testName: "login", + cluster_id: "c1", rca_done: "resolved", confidence: "high", - coverage: "full", root_cause: "PR #7421 tightened validator", - related_prs: "#7421", }, { testRunId: "102", - testName: "checkout", - rca_done: "blocked", + cluster_id: "c1", + rca_done: "failed", confidence: "", - coverage: "", root_cause: "", - related_prs: "", }, ]; - const md = renderReport(rows, { buildId: "b1" }); - assert.match(md, /2 test\(s\)/); - assert.match(md, /resolved: 1/); - assert.match(md, /blocked: 1/); - assert.match(md, /101/); - assert.match(md, /not available/); // 102's blank fields degrade + const txt = renderGlimpse(rows, { buildId: "b1" }); + assert.match(txt, /2 test\(s\)/); + assert.match(txt, /resolved: 1/); + assert.match(txt, /failed: 1/); + assert.match(txt, /101 → c1 → resolved → high: PR #7421 tightened validator/); + assert.match(txt, /102 → c1 → failed → -/); // blank fields degrade to "-" }); -test("report escapes pipes and collapses newlines in cells", () => { +test("glimpse one-liner truncates and collapses newlines (stays terse)", () => { const rows = [ { testRunId: "1", - testName: "t", + cluster_id: "solo-1", rca_done: "resolved", - root_cause: "a | b\nsecond line", - related_prs: "#1", + confidence: "medium", + root_cause: `line one\nline two ${"x".repeat(200)}`, }, ]; - const md = renderReport(rows); - assert.ok(!md.includes("a | b\nsecond")); - assert.match(md, /a \\\| b second line/); -}); - -test("report surfaces coverage caveats for thin/partial rows", () => { - const rows = [ - { testRunId: "1", testName: "t", rca_done: "resolved", coverage: "partial" }, - ]; - const md = renderReport(rows); - assert.match(md, /Coverage caveats/); - assert.match(md, /confidence band reflects evidence that was unavailable/); + const txt = renderGlimpse(rows); + const line = txt.split("\n").find((l) => l.startsWith("1 →")); + assert.ok(line.includes("line one line two")); + assert.ok(line.endsWith("…")); + assert.ok(line.length < 120); }); diff --git a/tests/fixtures/recorded-turns/blocked.json b/tests/fixtures/recorded-turns/blocked.json deleted file mode 100644 index 35b5373..0000000 --- a/tests/fixtures/recorded-turns/blocked.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "blocked — unmet asks", - "testRunId": 72, - "turns": [ - { - "status": "BLOCKED", - "confidence": "low", - "threadId": "thr-72", - "reason": "could not obtain server-side logs; cannot distinguish product bug from env flake", - "unmetAsks": ["kibana", "k8s"] - } - ] -} diff --git a/tests/fixtures/recorded-turns/pending.json b/tests/fixtures/recorded-turns/pending.json index ec8b16a..2e73ce7 100644 --- a/tests/fixtures/recorded-turns/pending.json +++ b/tests/fixtures/recorded-turns/pending.json @@ -1,10 +1,9 @@ { - "name": "soft-pending — resumable", + "name": "soft-pending — resumable (trimmed shape: status/threadId/turnId only)", "testRunId": 81, "turns": [ { "status": "PENDING", - "confidence": "unknown", "threadId": "thr-81", "turnId": "turn-81-1" } diff --git a/tests/fixtures/recorded-turns/resolved.json b/tests/fixtures/recorded-turns/resolved.json index 120dad6..47b9034 100644 --- a/tests/fixtures/recorded-turns/resolved.json +++ b/tests/fixtures/recorded-turns/resolved.json @@ -1,5 +1,5 @@ { - "name": "needs_info → evidence → resolved", + "name": "needs_info → evidence → resolved (trimmed terminal glimpse)", "testRunId": 39, "turns": [ { @@ -26,12 +26,12 @@ "status": "RESOLVED", "confidence": "high", "threadId": "thr-39", - "rca": { + "glimpse": { "root_cause": "PR #7421 tightened the buildName validator to reject empty strings", - "possible_fix": "send a non-empty buildName or relax the validator", "failure_type": "product_regression", "related_prs": ["#7421"] - } + }, + "viewRca": "https://observability.browserstack.com/builds/awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2" } ] } From 412611938a9986d91f38d0af420f8dd35af93355 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Mon, 13 Jul 2026 00:28:41 +0530 Subject: [PATCH 21/33] feat(rca): coordinator + batch workflow go autonomous on trimmed shapes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ai-tfa-coordinator: drop the mode input and the interactive gap-return (GAP_OUTPUT); gaps always degrade to 'unavailable' — never a prompt. Classify only RESOLVED|PENDING|NEEDS_INFO (BLOCKED gone); consume the trimmed RESOLVED glimpse {root_cause, failure_type, related_prs} + viewRca and pass it through verbatim; RCA_OUTPUT gains failure_type + view_rca, drops possible_fix/BLOCKED - mandatory culprit-PR hunt for PRODUCT_BUG / application bugs: feed PR link(s) to TFA so dashboard related_prs populates; no PR by turn cap → explicit 'no culprit PR identified after <searched>' + recorded gap - workflows/rca-batch.mjs: single autonomous path (no mode arg), schema matches the new RCA_OUTPUT, PRODUCT_BUG mandate in the shared prompt Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- agents/ai-tfa-coordinator.md | 144 ++++++++++++++++++----------------- workflows/rca-batch.mjs | 35 +++++---- 2 files changed, 96 insertions(+), 83 deletions(-) diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 3de14aa..167b155 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -1,9 +1,9 @@ --- name: ai-tfa-coordinator -description: 'Per-test collaborative-RCA coordinator. Given ONE testRunId, drives the tfaRcaTurn MCP loop to a terminal root cause: TFA reads the run logs; this coordinator supplies every non-log evidence ask (product code, k8s, kibana, metrics, deploy, ci) using whatever skills/tools the client has, routed through the capability manifest. Skips every test_logs ask (TFA owns logs). Emits a structured RCA_OUTPUT block. Generic over product and infra — no hardcoded tools. Examples: +description: 'Per-test collaborative-RCA coordinator (autonomous — never prompts a user). Given ONE testRunId, drives the tfaRcaTurn MCP loop to a terminal root cause: TFA reads the run logs; this coordinator supplies every non-log evidence ask (product code, k8s, kibana, metrics, deploy, ci) using whatever skills/tools the client has, routed through the validated capability manifest. Skips every test_logs ask (TFA owns logs). For application bugs it MUST hunt the culprit PR via the github connector. Emits a structured RCA_OUTPUT block. Generic over product and infra — no hardcoded tools. Examples: - orchestrator: Agent(subagent_type="tfa-rca:ai-tfa-coordinator", prompt="RCA testRunId=39 — error: empty buildName rejected on POST /builds") → drives the loop, returns RCA_OUTPUT - sibling confirm: Agent(subagent_type="tfa-rca:ai-tfa-coordinator", prompt="RCA testRunId=40 — pre-seed: cause=<rep root cause>, suspect PR=#7421") → one-turn confirm against this test logs -- user: "run collaborative RCA on test run 39" → single-test loop to RESOLVED/BLOCKED/PENDING' +- user: "run collaborative RCA on test run 39" → single-test loop to RESOLVED/PENDING' tools: [Bash, Read, Grep, Glob, Task, mcp__*__tfaRcaTurn, mcp__github__*] model: sonnet --- @@ -15,13 +15,19 @@ The collaboration contract is fixed: **TFA owns logs; this coordinator owns everything else.** TFA (server-side, via the tool) reads the run's logs from its own access and emits typed evidence asks; this coordinator fulfills every **non-log** ask using whatever skills/tools the client has — routed through the -capability manifest — digests the findings, and feeds them back on the same -thread until TFA converges. TFA authors the RCA into the TRA dashboard. +validated capability manifest — digests the findings, and feeds them back on the +same thread until TFA converges. TFA authors the RCA into the TRA dashboard; +this coordinator only ever sees the **trimmed glimpse** of it. The full report +lives on the Test Observability UI. + +This coordinator is **fully autonomous**: the `/factory` gate closed before it +was dispatched, so it **never prompts a user** — an evidence gap degrades to an +`unavailable` block back to TFA, always. This coordinator is the **reusable unit**: it takes one `testRunId` and runs -standalone, driven by the auto workflow, an interactive subagent, or a thin -sequential harness. It is **generic over product and infra** — it names no -`kubectl` / `chitragupta` / `bifrost`; it routes by *capability*. +standalone, driven by the batch workflow, a subagent dispatch, or the thin +sequential harness (`lib/loop.mjs`). It is **generic over product and infra** — +it names no `kubectl` / `chitragupta` / `bifrost`; it routes by *capability*. ## Inputs @@ -31,13 +37,25 @@ sequential harness. It is **generic over product and infra** — it names no `root_cause` + suspect `related_prs`. When present, the first-turn message states the hypothesis and asks TFA to **confirm it against this test's own logs**. - `resume` — optional `{ threadId, turnId }` from a prior PENDING run. -- `manifest` — the capability manifest `{ capability: { available, via } }` (from the orchestrator's pre-compute). -- `mode` — `auto` | `interactive`. Selects the **gap-resolver** (see below). +- `manifest` — the validated capability manifest `{ capability: { available, via } }` + (built once at the `/factory` gate — Part A). If `testRunId` is missing or not parseable as an integer, emit a `failed` `RCA_OUTPUT` block with `root_cause: "no testRunId provided"` and stop — do not call the tool. +## What the tool returns (trimmed shapes) + +`tfaRcaTurn` returns **trimmed** terminal turns — never the full RCA payload: + +- `RESOLVED` → `{ status, confidence, threadId, glimpse: { root_cause (≤220 + chars), failure_type, related_prs }, viewRca }`. The `viewRca` link points at + the Test Observability UI — pass it through to the output. +- `PENDING` → `{ status, turnId, threadId }` (soft-pending; in-call poll + exceeded its wall-clock cap — resumable). +- `NEEDS_INFO` → `questions` / `asks` / `suggestions` **verbatim** — this loop + consumes them exactly as sent. + ## Operating principles 1. **Logs by TFA — the core contract.** Never seed logs in the first turn; @@ -50,54 +68,37 @@ call the tool. extra turn, never a busy-wait. 4. **One thread per test.** First turn omits `threadId`; capture it from the response and reuse it on every follow-up. Never start a second thread. -5. **Soft-PENDING ends the loop.** A tool result of `status: "PENDING"` (in-call - poll exceeded its wall-clock cap) ends the loop immediately as `PENDING`, - carrying `threadId` + `turnId` for a later resume. Do not re-poll or sleep. +5. **Soft-PENDING ends the loop.** A tool result of `status: "PENDING"` ends the + loop immediately as `PENDING`, carrying `threadId` + `turnId` for a later + resume. Do not re-poll or sleep. 6. **Digest, don't dump.** Every follow-up `message` carries digested findings (`ask → found → snippet/link`), never raw log tails, full diffs, or full files. Size caps + block shape live in `references/evidence-routing.md` — read it before fulfilling any ask. The tool caps `message` at 5000 chars. 7. **Report gaps, don't drop them.** An ask the coordinator cannot fulfill becomes - a `not-found` / `unreachable` / `unavailable` block, never a silent omission. + a `not-found` / `unreachable` / `unavailable` block, never a silent omission — + and **never a user prompt**. TFA finalizes best-effort with lower confidence. 8. **Never editorialize.** Report findings (suspect PR, server-side error line), - not verdicts. The root cause is TFA's to state on `RESOLVED`; pass its `rca` - through verbatim. - -## The gap-resolver (mode fork) - -Routing an ask yields `skip` / `gather` / `gap` (see `references/evidence-routing.md`). -The only behavioral difference between modes is what happens on a **gap** (no -capability available for that `evidenceType`): - -- **auto** → emit an `unavailable` block back to TFA (no user prompt). TFA - finalizes best-effort with lower confidence. -- **interactive** → a subagent cannot pause to prompt the user, so **end the run - early and return a `GAP_OUTPUT` block** (status `PENDING`) carrying the resume - handles + the gap. The orchestrator asks A1, then **re-dispatches a coordinator - with `resume={threadId, turnId}`** and the answer digested into the next turn. - See `references/interactive-mode.md`. - -`GAP_OUTPUT` block (interactive gap only): - -``` -GAP_OUTPUT_START -## testRunId -<integer> -## thread_id -<threadId> -## turn_id -<turnId> # resume handle -## gap -- evidenceType: <type> -- what: <verbatim ask `what`> -- why: <verbatim ask `why`> -GAP_OUTPUT_END -``` - -Everything else — the loop, routing, digest, caps, terminal output — is identical -across modes. Do not fork the loop; only the gap action differs. When all gaps in -a turn are resolvable (gathered or user-answered), the loop proceeds normally to a -terminal `RCA_OUTPUT`. + not verdicts. The root cause is TFA's to state on `RESOLVED`; pass its + `glimpse` through verbatim. + +## Application bugs — the culprit-PR mandate (MANDATORY) + +Whenever TFA's classification (in an ask, a suggestion, or the resolving +`glimpse.failure_type`) is **PRODUCT_BUG / application bug**, the github +connector is the deliverable, not optional evidence: + +- **Hunt the culprit PR**: deploy timeline vs the last-pass window, changed + paths vs the failure signature (`references/github-evidence.md`), run the + falsification protocol on each candidate. +- **Feed the PR link(s) to TFA in the turn message** so the BrowserStack agent + populates `related_prs` in the dashboard RCA. +- **An application-bug RCA with no GitHub PR link is INCOMPLETE.** Keep digging + on subsequent turns until the turn cap. If still none, the turn message must + explicitly state `no culprit PR identified after <what was searched: window, + repos, paths>` — and the orchestrator records the gap on the CSV row. +- If the github connector is invalid/absent (a gate-recorded gap), state the + same explicitly plus an `unavailable` block. Never fabricate a PR. ## Suspect-PR falsification (github asks) @@ -122,8 +123,7 @@ re-fetch per test. Never fabricate a PR when the github capability is unavailabl 1. SUBMIT turn 1: tfaRcaTurn(testRunId=<id>, message=<digest>). Capture threadId. turns_used = 1. (resume case: tfaRcaTurn(testRunId, threadId, turnId) instead, then continue at 2.) 2. CLASSIFY result.status: - RESOLVED → capture rca; END (RESOLVED). - BLOCKED → capture reason + unmetAsks; END (BLOCKED). + RESOLVED → capture glimpse + viewRca; END (RESOLVED). PENDING → capture threadId + turnId; END (PENDING, note "soft-pending"). NEEDS_INFO → go to 3. 3. ROUTE the asks (read references/evidence-routing.md; route via lib/routing.mjs): @@ -131,7 +131,8 @@ re-fetch per test. Never fabricate a PR when the github capability is unavailabl skip → record in asks_skipped, emit nothing. gather → run the discovered skill/tool for its capability, digest into one block. Record evidenceType in asks_fulfilled (dedupe). - gap → run the mode's gap-resolver (auto: unavailable block; interactive: return to caller). + gap → emit an `unavailable` block (record in asks_unavailable). NEVER prompt. + PRODUCT_BUG in play + no supported PR yet → widen the github hunt this turn. Concatenate per-ask blocks into the next-turn MESSAGE (respect size caps). 4. SUBMIT follow-up on the SAME thread: tfaRcaTurn(testRunId, message, threadId). turns_used += 1. 5. TURN-CAP CHECK: if turns_used >= turnCap and still NEEDS_INFO → END (PENDING, "turn-cap"). @@ -142,20 +143,20 @@ re-fetch per test. Never fabricate a PR when the github capability is unavailabl > The loop mechanics above have an **executable mirror** in `lib/loop.mjs` > (`runRcaLoop`) — conformance-tested against recorded `tfaRcaTurn` transcripts > (`tests/conformance.test.mjs`). It also serves as the **sequential thin-client -> harness** (D5): MCP clients without workflows/subagents drive the same contract +> harness**: MCP clients without workflows/subagents drive the same contract > by calling `runRcaLoop` with a real `submit` bound to `tfaRcaTurn`. **Sibling confirm (cluster member).** When `pre_seed` is present the first turn states the representative's hypothesis and asks TFA to confirm against this test's own logs. If TFA `RESOLVED`s in one turn → a logs-grounded per-test RCA at -minimal cost. If TFA instead returns `NEEDS_INFO` / `BLOCKED` (the hypothesis -does not hold for this test), **fall back to the normal loop** — never blindly -inherit the representative's cause. +minimal cost. If TFA instead returns `NEEDS_INFO` (the hypothesis does not hold +for this test), **fall back to the normal loop** — never blindly inherit the +representative's cause. ## Output contract — `RCA_OUTPUT` Emit **exactly one** block at the end of every run (including the `failed` -no-input case). The orchestrator parses it into one CSV row / report record. +no-input case). The orchestrator parses it into one CSV row / glimpse line. ``` RCA_OUTPUT_START @@ -164,19 +165,22 @@ RCA_OUTPUT_START <integer> ## status -<RESOLVED | BLOCKED | PENDING | failed> +<RESOLVED | PENDING | failed> ## confidence <high | medium | low | unknown> # from the terminal turn; unknown for PENDING/failed ## root_cause -<RESOLVED → rca.root_cause verbatim · BLOCKED → TFA's reason · PENDING/failed → "not available" or the note> +<RESOLVED → glimpse.root_cause verbatim (already ≤220 chars) · PENDING/failed → "not available" or the note> -## possible_fix -<RESOLVED → rca.possible_fix verbatim · else "not available"> +## failure_type +<RESOLVED → glimpse.failure_type verbatim · else "not available"> ## related_prs -- <each PR TFA recorded in rca.related_prs; "none" if empty> +- <each PR in glimpse.related_prs; "none" if empty — for PRODUCT_BUG, "none" only after the mandated hunt + explicit statement> + +## view_rca +<viewRca link from the RESOLVED turn (Test Observability UI) · "not available" if none> ## suspect_signals - <each non-log signal surfaced: suspect PR / deploy / server-side error line; "none" if empty> @@ -197,28 +201,32 @@ RCA_OUTPUT_START - test_logs # present once a test_logs ask appeared ## asks_unavailable -- <evidenceType> # gaps with no capability (drives the coverage stamp, U10); "none" if empty +- <evidenceType> # gate-recorded gaps (drives the coverage stamp); "none" if empty RCA_OUTPUT_END ``` Notes: -- `status` is one of exactly four values. `turn-cap` and `soft-pending` both +- `status` is one of exactly three values. `turn-cap` and `soft-pending` both report as `PENDING`; note which in `root_cause`. - `asks_skipped` always includes `test_logs` whenever TFA asked for logs. `asks_fulfilled` **never** includes `test_logs`. -- `asks_unavailable` is the evidence-coverage signal U10 turns into a confidence band. +- `asks_unavailable` is the evidence-coverage signal the coverage stamp turns + into a confidence band. - `failed` is the no-parseable-result / no-input case; the orchestrator synthesizes a `failed` row if this coordinator dies — keep the block valid. ## Hard limits +- **Never** prompt, ask, or wait on a user — the gate is closed; gaps degrade to `unavailable`. - **Never** fulfill or seed a `test_logs` ask — TFA owns logs. - **Never** exceed `turnCap` `tfaRcaTurn` calls in one run. - **Never** start a second thread for the same test — reuse the first turn's `threadId`. - **Never** busy-wait / re-poll on a soft-`PENDING` — end and report it resumable. - **Never** dump raw logs, full diffs, or full file contents into a turn message — digest only. - **Never** write to any repo / cluster / ticket / the run — every action is read-only. -- **Never** editorialize a cause — pass TFA's `rca` through verbatim. +- **Never** editorialize a cause — pass TFA's `glimpse` through verbatim. - **Never** blindly inherit a representative's cause for a sibling — confirm against its own logs. +- **Never** resolve an application bug silently without a PR link — hunt until the + turn cap, else state "no culprit PR identified after <searched>" explicitly. - **Always** emit exactly one valid `RCA_OUTPUT` block, even on the `failed` path. diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs index 07868e9..1fe8699 100644 --- a/workflows/rca-batch.mjs +++ b/workflows/rca-batch.mjs @@ -1,24 +1,27 @@ export const meta = { name: "rca-batch", description: - "Drive collaborative RCA over all failed tests of a build (auto mode): cluster representatives run the full loop, siblings one-turn-confirm, ~5 concurrent.", + "Drive autonomous collaborative RCA over all failed tests of a build: cluster representatives run the full loop, siblings one-turn-confirm, ~5 concurrent. Never prompts a user.", phases: [ { title: "Representatives", detail: "full multi-turn RCA per cluster" }, { title: "Siblings", detail: "one-turn confirm against own logs" }, ], }; -// AUTO MODE orchestration (D2). This is a dynamic-workflow script: it runs in the -// Workflow sandbox (no filesystem, no Date.now/Math.random, agent()/pipeline() -// as globals). It therefore does NO state I/O itself — the orchestrator seeds the -// CSV, clusters, and builds the manifest in normal context and passes the -// work-list via `args`; each dispatched `ai-tfa-coordinator` agent (which HAS -// tool access) claims + flips its own CSV row eagerly (WAL); this script -// orchestrates concurrency and returns the structured results for reconciliation. +// The /factory batch orchestration (fully autonomous — the gate closed before +// this runs; nothing here ever asks the user). This is a dynamic-workflow +// script: it runs in the Workflow sandbox (no filesystem, no Date.now/ +// Math.random, agent()/pipeline() as globals). It therefore does NO state I/O +// itself — the orchestrator seeds the CSV, clusters, and builds the validated +// manifest at the gate and passes the work-list via `args`; each dispatched +// `ai-tfa-coordinator` agent (which HAS tool access) claims + flips its own CSV +// row eagerly (WAL); this script orchestrates concurrency and returns the +// structured results for reconciliation. The final glimpse + triggerRcaReport +// step happens back in the orchestrator (SKILL.md Step 6). // // args shape: // { -// csvPath, buildId, mode: "auto", +// csvPath, buildId, // manifest: { capability: { available, via } }, // buildEvidence: { baselineRef, suspectWindow, ... }, // pre-computed once // clusters: [ @@ -32,10 +35,11 @@ const RCA_SCHEMA = { required: ["testRunId", "status"], properties: { testRunId: { type: "string" }, - status: { enum: ["RESOLVED", "BLOCKED", "PENDING", "failed"] }, + status: { enum: ["RESOLVED", "PENDING", "failed"] }, confidence: { enum: ["high", "medium", "low", "unknown"] }, root_cause: { type: "string" }, - possible_fix: { type: "string" }, + failure_type: { type: "string" }, + view_rca: { type: "string" }, related_prs: { type: "array", items: { type: "string" } }, suspect_signals: { type: "array", items: { type: "string" } }, threadId: { type: "string" }, @@ -55,7 +59,8 @@ const shared = [ `CSV state file: ${ctx.csvPath}`, `Capability manifest: ${JSON.stringify(ctx.manifest ?? {})}`, `Build-level evidence (pre-computed once, reuse — do not re-fetch): ${JSON.stringify(ctx.buildEvidence ?? {})}`, - `Mode: auto — on an evidence gap with no capability, report "unavailable" back to TFA (NEVER prompt a user). Best-effort finalize.`, + `Autonomous run — on an evidence gap with no valid connector, report "unavailable" back to TFA (NEVER prompt a user). Best-effort finalize.`, + `PRODUCT_BUG / application-bug mandate: hunt the culprit PR via the github connector (deploy timeline vs last-pass window, changed paths vs failure signature) and feed the PR link(s) to TFA so related_prs populates. No PR after digging to the turn cap → state explicitly "no culprit PR identified after <what was searched>" so the CSV row records the gap.`, `Persist eagerly to the CSV: claim your row before turn 1, flip it on terminal (lib/csv-state.mjs).`, ].join("\n"); @@ -78,7 +83,7 @@ function siblingPrompt(sibling, repResult, cluster) { ` root_cause: ${repResult?.root_cause ?? "(representative did not resolve)"}`, ` related_prs: ${JSON.stringify(repResult?.related_prs ?? [])}`, `State this hypothesis on turn 1 and ask TFA to CONFIRM it against THIS test's own logs.`, - `If TFA confirms in one turn → done. If it does NOT (NEEDS_INFO/BLOCKED), fall back to the full loop — never blindly inherit.`, + `If TFA confirms in one turn → done. If it does NOT (NEEDS_INFO), fall back to the full loop — never blindly inherit.`, `testRunId=${sibling.testRunId} testName=${sibling.testName ?? ""}`, `error_digest: ${sibling.error_summary ?? "(none)"}`, shared, @@ -86,7 +91,7 @@ function siblingPrompt(sibling, repResult, cluster) { ].join("\n"); } -log(`Auto-mode batch: ${clusters.length} cluster(s) over build ${ctx.buildId ?? "?"}`); +log(`Batch: ${clusters.length} cluster(s) over build ${ctx.buildId ?? "?"}`); // Pipeline: each cluster flows representative → siblings independently (no barrier // between stages), so a small cluster's siblings confirm while a big cluster's @@ -125,6 +130,6 @@ const byStatus = all.reduce((acc, r) => { return acc; }, {}); -log(`Auto-mode batch complete: ${all.length} test(s) — ${JSON.stringify(byStatus)}`); +log(`Batch complete: ${all.length} test(s) — ${JSON.stringify(byStatus)}`); return { clusters: flat.length, tests: all.length, byStatus, results: flat }; From 97dbe1d6db9db1422a5dfac8bbfda3863f099bf1 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Mon, 13 Jul 2026 00:30:47 +0530 Subject: [PATCH 22/33] docs(rca): tell the single-gate /factory story; scrub mode/report/BLOCKED residue - README: /factory usage, the one-gate two-part flow (validated connector manifest + assumption-first intake, at most one consolidated question), autonomous execution, glimpse + triggerRcaReport + Test Observability UI link as the only outputs - INTEGRATION: factory skill naming, triggerRcaReport in the MCP core row, no-local-report finish on every host - automation/README: demo invocation is now /factory <build-id> - lib/routing.mjs comments: gaps degrade to 'unavailable', no resolveGap Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- INTEGRATION.md | 34 ++++++++------- README.md | 100 +++++++++++++++++++++++++++---------------- automation/README.md | 2 +- lib/routing.mjs | 13 +++--- 4 files changed, 90 insertions(+), 59 deletions(-) diff --git a/INTEGRATION.md b/INTEGRATION.md index 9d91cb5..e72cbf6 100644 --- a/INTEGRATION.md +++ b/INTEGRATION.md @@ -2,36 +2,40 @@ This plugin is built so the **MCP core is truly cross-client** and the **harness layer ports via the cross-vendor Agent Skills standard**. Only one piece is -genuinely Claude-Code-specific (the auto-mode *dynamic workflow*); on Cursor and -Codex that role is filled by the sequential harness or interactive subagents. +genuinely Claude-Code-specific (the batch *dynamic workflow*); on Cursor and +Codex that role is filled by the sequential harness or subagents. Every path is +autonomous after the single `/factory` gate — no host ever prompts mid-run. ## What transfers, what doesn't | Layer | Claude Code | Cursor | Codex | |---|---|---|---| -| `bstack` MCP server (`listTestIds` + `tfaRcaTurn`) | `.mcp.json` (auto-discovered) | `.cursor-mcp.json` / `.cursor/mcp.json` | `~/.codex/config.toml` `[mcp_servers.bstack]` | -| `rca-build` skill (`SKILL.md`) | plugin `skills/` | Agent Skills (`.cursor/skills/` or cursor-plugin `"skills":"./skills/"`) | Agent Skills (`.agents/skills/`) | +| `bstack` MCP server (`listTestIds` + `tfaRcaTurn` + `triggerRcaReport`) | `.mcp.json` (auto-discovered) | `.cursor-mcp.json` / `.cursor/mcp.json` | `~/.codex/config.toml` `[mcp_servers.bstack]` | +| `factory` skill (`SKILL.md`) | plugin `skills/` | Agent Skills (`.cursor/skills/` or cursor-plugin `"skills":"./skills/"`) | Agent Skills (`.agents/skills/`) | | `ai-tfa-coordinator` agent | plugin `agents/` | `.cursor/agents/` (also reads `.claude/agents/`) | `.codex/agents/` | | Per-test RCA **loop** | `agents/ai-tfa-coordinator.md` | same skill/agent | same skill/agent | -| Batch orchestration | **auto** = dynamic workflow `workflows/rca-batch.mjs`; **interactive** = subagents | subagents, or **sequential** `lib/loop.mjs` | subagents, or **sequential** `lib/loop.mjs` | +| Batch orchestration | dynamic workflow `workflows/rca-batch.mjs` (or subagents) | subagents, or **sequential** `lib/loop.mjs` | subagents, or **sequential** `lib/loop.mjs` | The dynamic workflow (`workflows/rca-batch.mjs`) uses Claude Code's Workflow runtime, which Cursor/Codex don't have. The same batch still runs there via -**interactive subagents** (both hosts support subagents) or the **sequential -thin-client harness** `lib/loop.mjs` (`runRcaLoop`) — the conformance-tested -loop that drives `tfaRcaTurn` over the same contract without any host-specific -orchestration. +**subagents** (both hosts support subagents) or the **sequential thin-client +harness** `lib/loop.mjs` (`runRcaLoop`) — the conformance-tested loop that +drives `tfaRcaTurn` over the same contract without any host-specific +orchestration. On every host the run finishes the same way: glimpse table → +`triggerRcaReport(buildUuid)` → "Full report on the Test Observability UI: +<viewReport>". No local report file is ever written. ## Claude Code ```bash cp .env.example .env # BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY claude --plugin-dir ./ -/rca-build <build-id> +/factory <build-id> ``` -`.claude-plugin/plugin.json` + root `.mcp.json` + `skills/` + `agents/` + -`commands/` are auto-discovered. +`.claude-plugin/plugin.json` + root `.mcp.json` + `skills/` + `agents/` are +auto-discovered. (No `commands/factory.md` on purpose — a command and skill +with the same name collide and the skill body fails to load.) ## Cursor @@ -59,7 +63,7 @@ ln -s ../skills .cursor/skills ln -s ../agents .cursor/agents ``` -Then drive it from Agent chat: invoke the `rca-build` skill with a build id. +Then drive it from Agent chat: invoke the `factory` skill with a build id. ## Codex @@ -84,7 +88,7 @@ ln -s ../skills .agents/skills ln -s ../agents .codex/agents ``` -Then run the `rca-build` skill; the coordinator + `tfaRcaTurn` loop are identical. +Then run the `factory` skill; the coordinator + `tfaRcaTurn` loop are identical. ## Notes @@ -95,4 +99,4 @@ Then run the `rca-build` skill; the coordinator + `tfaRcaTurn` loop are identica Cursor/Codex, replace the placeholders with literals if your client doesn't expand them. - Everything in `lib/` and the `SKILL.md`/agent prose is host-agnostic — only the - MCP wiring file and the auto-mode workflow are host-specific. + MCP wiring file and the dynamic workflow are host-specific. diff --git a/README.md b/README.md index 34a910c..88fb748 100644 --- a/README.md +++ b/README.md @@ -4,15 +4,18 @@ Drive BrowserStack's collaborative root-cause-analysis loop over **all failed tests of a build**, generic across product and infra, from inside an agentic MCP client (Claude Code / Cursor / Codex). -The plugin wraps two stable MCP tools — `listTestIds` and `tfaRcaTurn` (from the -`bstack` MCP server) — and adds the harness that batches RCA over a whole build, -clusters failures by signature, routes evidence requests to whatever -skills/tools the client already has, and writes a per-test RCA into the TRA -dashboard. - -> It **discovers and delegates** to the infra skills/tools already in your -> client (GitHub, k8s/EKS, kibana/other logs, metrics). It does **not** install -> or own those connectors. +The plugin wraps three stable MCP tools — `listTestIds`, `tfaRcaTurn`, and +`triggerRcaReport` (from the `bstack` MCP server) — and adds the harness that +batches RCA over a whole build, clusters failures by signature, routes evidence +requests to whatever skills/tools the client already has, and lands a per-test +RCA in the TRA (Test Observability) dashboard. + +> **The full RCA report lives on the Test Observability UI, not in Claude.** +> The plugin surfaces a terse glimpse, triggers the dashboard report +> (`triggerRcaReport`), and prints the link. It **discovers and delegates** to +> the infra skills/tools already in your client (GitHub, k8s/EKS, kibana/other +> logs, metrics). It does **not** install or own those connectors, and it never +> writes a local report file. ## Install @@ -24,42 +27,63 @@ claude --plugin-dir ./ ``` The plugin auto-configures on load: the `bstack` MCP server (from `.mcp.json`), -the `/rca-build` command, the `rca-build` skill, and the `ai-tfa-coordinator` -agent are all discovered by convention. +the `factory` skill, and the `ai-tfa-coordinator` agent are all discovered by +convention. (There is deliberately **no** command file named `factory` — a +command and skill sharing a name collide and the skill body fails to load.) ### Cursor & Codex -The MCP core (`listTestIds` + `tfaRcaTurn`) and the skill/agent layer port to -both — Cursor uses `.cursor-plugin/plugin.json` + `.cursor-mcp.json`, Codex uses -`~/.codex/config.toml` (see `codex-mcp.example.toml`). The only Claude-specific -piece is the auto-mode *dynamic workflow*; on Cursor/Codex the same batch runs via -interactive subagents or the sequential harness (`lib/loop.mjs`). Full -per-host wiring (MCP config, skill/agent discovery, deeplink) is in -**[INTEGRATION.md](INTEGRATION.md)**. +The MCP core (`listTestIds` + `tfaRcaTurn` + `triggerRcaReport`) and the +skill/agent layer port to both — Cursor uses `.cursor-plugin/plugin.json` + +`.cursor-mcp.json`, Codex uses `~/.codex/config.toml` (see +`codex-mcp.example.toml`). The only Claude-specific piece is the batch *dynamic +workflow*; on Cursor/Codex the same batch runs via subagents or the sequential +harness (`lib/loop.mjs`). Full per-host wiring (MCP config, skill/agent +discovery, deeplink) is in **[INTEGRATION.md](INTEGRATION.md)**. ## Usage ``` -/rca-build <build-id> -/rca-build build_id=<id> mode=auto +/factory <build-id> +/factory build_id=<id> https://github.com/org/repo/pull/123 ``` -On start the plugin runs a **mandatory pre-flight intake** asking for your -product + automation repos, working branch, default branch, and the PRs in -play, plus the build id. Every question is answerable with "I don't have one" → -the run proceeds RCA-only. +Args: a build id (bare, `build_id=`, or a dashboard link) plus optional PR URLs +/ repo hints. + +## The single gate + +The run has exactly **one gate** before execution, with two parts: + +1. **Connector discovery + validation** — every connector relevant to test RCA + (github, k8s, logs, metrics, …) is enumerated and probe-validated (`gh auth + status`, `kubectl` reachability, MCP tools listed). The result is a validated + capability manifest: `connector → valid | invalid | absent`. A gap is + recorded and declared to the TFA agent ("I don't have logs/metrics access") — + never a blocker. +2. **Requirements** — intake fields (product repo, automation repo, branches, + PRs in play, build id) are resolved **by assumption** wherever possible + (invocation args, `gh repo view`, current branch). At most **one** + consolidated question may be asked at gate close, and only for genuinely + non-assumable, load-bearing fields. Headless (`claude -p`) never asks: a + missing build id fails fast; everything else is a recorded gap. -## Modes +**After the gate closes, the run never asks you anything again** — RCA +execution is fully autonomous. Evidence gaps degrade to "unavailable" back to +the TFA agent, which finalizes best-effort. -- **auto** — a dynamic workflow drives the whole batch (5 tests concurrent), no - mid-run prompts. When evidence can't be gathered (no matching skill), it - reports "unavailable" back to the TFA agent, which finalizes best-effort. -- **interactive** — the main session spawns subagents (5 at a time); on an - evidence gap a subagent returns the gap to the main agent, which asks you, - then feeds the answer back. +## Output + +When every test is terminal, the run prints a terse **glimpse table** +(`testRunId → cluster → status → confidence one-liner`), calls +`triggerRcaReport(buildUuid)`, and prints: + +``` +Full report on the Test Observability UI: <viewReport link> +``` -`auto` means autonomy *during* the batch from an interactive session — not -headless. Running `claude -p` with a required input missing ends immediately. +That dashboard report — populated per-test by the BrowserStack agent, with +mandatory culprit-PR links on application bugs — is the real deliverable. ## Requirements @@ -83,11 +107,13 @@ A seeded failing build exercises the full loop against real staging infra: 3. **Run** — with `BROWSERSTACK_USERNAME`/`ACCESS_KEY` exported and `kubectl` pointed at the staging cluster: ``` - /rca-build awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2 mode=interactive + /factory awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2 ``` - The harness clusters the failures, drives `tfaRcaTurn` per cluster, and routes - `k8s` asks to the `k8s-rengg-tfa` skill while `product_code`/`deploy` asks go to - GitHub — landing per-test RCAs that trace back to the seeded regressions. + The gate validates connectors (github via `gh`, k8s via the `k8s-rengg-tfa` + skill), then the harness clusters the failures, drives `tfaRcaTurn` per + cluster, routes `k8s` asks to the skill while `product_code`/`deploy` asks go + to GitHub — landing per-test RCAs on the dashboard that trace back to the + seeded regressions, then prints the glimpse + the Test Observability link. ## Layout diff --git a/automation/README.md b/automation/README.md index fa84eb2..d58f5bd 100644 --- a/automation/README.md +++ b/automation/README.md @@ -31,7 +31,7 @@ PROJECT_NAME="RCA Feature Fencing" BUILD_NAME="VRT Build" \ The upload response carries the build id. Feed it to the plugin: ``` -/rca-build <build-id> mode=interactive +/factory <build-id> ``` ## The failures (what RCA should rediscover) diff --git a/lib/routing.mjs b/lib/routing.mjs index ec7c4e7..019a012 100644 --- a/lib/routing.mjs +++ b/lib/routing.mjs @@ -1,6 +1,7 @@ // Evidence-routing registry (D3). Maps a TFA `ask.evidenceType` onto an -// action, given the run's capability manifest. Pure + dependency-free so it is -// testable and reusable by both the auto workflow and interactive subagents. +// action, given the run's validated capability manifest. Pure + dependency-free +// so it is testable and reusable by the batch workflow, subagents, and the +// sequential harness alike. // // `test_logs` is the TFA agent's own evidence and is always skipped. Every // other type routes to a capability; whether that capability is *available* is @@ -28,8 +29,8 @@ export function orderAsks(asks = []) { // Classify one ask. Returns one of: // { action: "skip", ... } — test_logs / TFA-owned; the coordinator emits nothing // { action: "gather", ... } — a capability is available; gather + digest -// { action: "gap", ... } — no capability; the caller's resolveGap() decides -// (auto → "unavailable" block; interactive → ask the user) +// { action: "gap", ... } — no valid connector; the caller emits an +// "unavailable" block back to TFA (never a prompt) // // `manifest` shape: { [capability]: { available: boolean, via?: string } }. export function routeAsk(ask, config, manifest = {}) { @@ -62,8 +63,8 @@ export function routeAsk(ask, config, manifest = {}) { } // Split a turn's asks into the three buckets, in priority order. The -// coordinator gathers `gather`, runs resolveGap() on each `gap`, and records -// `skip` (test_logs) without emitting anything. +// coordinator gathers `gather`, emits an "unavailable" block for each `gap`, +// and records `skip` (test_logs) without emitting anything. export function routeAsks(asks, config, manifest = {}) { const ordered = orderAsks(asks); const buckets = { skip: [], gather: [], gap: [] }; From d2faebd1a13243224219f614d40ec2075f598a58 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Mon, 13 Jul 2026 00:38:18 +0530 Subject: [PATCH 23/33] =?UTF-8?q?fix(rca):=20keep=20the=20skill=20named=20?= =?UTF-8?q?rca-build=20=E2=80=94=20/factory=20is=20the=20user's=20spec-gen?= =?UTF-8?q?erator=20skill?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert only the rename from the single-gate rework (name collision with the user-level /factory skill); all auto-only, single-gate, glimpse, and PRODUCT_BUG-mandate behavior stays. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- INTEGRATION.md | 12 ++++++------ README.md | 10 +++++----- agents/ai-tfa-coordinator.md | 4 ++-- automation/README.md | 2 +- config/rca.config.json | 2 +- lib/glimpse.mjs | 2 +- lib/loop.mjs | 2 +- skills/{factory => rca-build}/SKILL.md | 4 ++-- .../{factory => rca-build}/references/clustering.md | 0 .../references/evidence-routing.md | 0 .../references/github-evidence.md | 0 workflows/rca-batch.mjs | 2 +- 12 files changed, 20 insertions(+), 20 deletions(-) rename skills/{factory => rca-build}/SKILL.md (99%) rename skills/{factory => rca-build}/references/clustering.md (100%) rename skills/{factory => rca-build}/references/evidence-routing.md (100%) rename skills/{factory => rca-build}/references/github-evidence.md (100%) diff --git a/INTEGRATION.md b/INTEGRATION.md index e72cbf6..65de0b6 100644 --- a/INTEGRATION.md +++ b/INTEGRATION.md @@ -4,14 +4,14 @@ This plugin is built so the **MCP core is truly cross-client** and the **harness layer ports via the cross-vendor Agent Skills standard**. Only one piece is genuinely Claude-Code-specific (the batch *dynamic workflow*); on Cursor and Codex that role is filled by the sequential harness or subagents. Every path is -autonomous after the single `/factory` gate — no host ever prompts mid-run. +autonomous after the single `/rca-build` gate — no host ever prompts mid-run. ## What transfers, what doesn't | Layer | Claude Code | Cursor | Codex | |---|---|---|---| | `bstack` MCP server (`listTestIds` + `tfaRcaTurn` + `triggerRcaReport`) | `.mcp.json` (auto-discovered) | `.cursor-mcp.json` / `.cursor/mcp.json` | `~/.codex/config.toml` `[mcp_servers.bstack]` | -| `factory` skill (`SKILL.md`) | plugin `skills/` | Agent Skills (`.cursor/skills/` or cursor-plugin `"skills":"./skills/"`) | Agent Skills (`.agents/skills/`) | +| `rca-build` skill (`SKILL.md`) | plugin `skills/` | Agent Skills (`.cursor/skills/` or cursor-plugin `"skills":"./skills/"`) | Agent Skills (`.agents/skills/`) | | `ai-tfa-coordinator` agent | plugin `agents/` | `.cursor/agents/` (also reads `.claude/agents/`) | `.codex/agents/` | | Per-test RCA **loop** | `agents/ai-tfa-coordinator.md` | same skill/agent | same skill/agent | | Batch orchestration | dynamic workflow `workflows/rca-batch.mjs` (or subagents) | subagents, or **sequential** `lib/loop.mjs` | subagents, or **sequential** `lib/loop.mjs` | @@ -30,11 +30,11 @@ orchestration. On every host the run finishes the same way: glimpse table → ```bash cp .env.example .env # BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY claude --plugin-dir ./ -/factory <build-id> +/rca-build <build-id> ``` `.claude-plugin/plugin.json` + root `.mcp.json` + `skills/` + `agents/` are -auto-discovered. (No `commands/factory.md` on purpose — a command and skill +auto-discovered. (No `commands/rca-build.md` on purpose — a command and skill with the same name collide and the skill body fails to load.) ## Cursor @@ -63,7 +63,7 @@ ln -s ../skills .cursor/skills ln -s ../agents .cursor/agents ``` -Then drive it from Agent chat: invoke the `factory` skill with a build id. +Then drive it from Agent chat: invoke the `rca-build` skill with a build id. ## Codex @@ -88,7 +88,7 @@ ln -s ../skills .agents/skills ln -s ../agents .codex/agents ``` -Then run the `factory` skill; the coordinator + `tfaRcaTurn` loop are identical. +Then run the `rca-build` skill; the coordinator + `tfaRcaTurn` loop are identical. ## Notes diff --git a/README.md b/README.md index 88fb748..9a3e170 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,8 @@ claude --plugin-dir ./ ``` The plugin auto-configures on load: the `bstack` MCP server (from `.mcp.json`), -the `factory` skill, and the `ai-tfa-coordinator` agent are all discovered by -convention. (There is deliberately **no** command file named `factory` — a +the `rca-build` skill, and the `ai-tfa-coordinator` agent are all discovered by +convention. (There is deliberately **no** command file named `rca-build` — a command and skill sharing a name collide and the skill body fails to load.) ### Cursor & Codex @@ -44,8 +44,8 @@ discovery, deeplink) is in **[INTEGRATION.md](INTEGRATION.md)**. ## Usage ``` -/factory <build-id> -/factory build_id=<id> https://github.com/org/repo/pull/123 +/rca-build <build-id> +/rca-build build_id=<id> https://github.com/org/repo/pull/123 ``` Args: a build id (bare, `build_id=`, or a dashboard link) plus optional PR URLs @@ -107,7 +107,7 @@ A seeded failing build exercises the full loop against real staging infra: 3. **Run** — with `BROWSERSTACK_USERNAME`/`ACCESS_KEY` exported and `kubectl` pointed at the staging cluster: ``` - /factory awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2 + /rca-build awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2 ``` The gate validates connectors (github via `gh`, k8s via the `k8s-rengg-tfa` skill), then the harness clusters the failures, drives `tfaRcaTurn` per diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 167b155..49ec5d4 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -20,7 +20,7 @@ same thread until TFA converges. TFA authors the RCA into the TRA dashboard; this coordinator only ever sees the **trimmed glimpse** of it. The full report lives on the Test Observability UI. -This coordinator is **fully autonomous**: the `/factory` gate closed before it +This coordinator is **fully autonomous**: the `/rca-build` gate closed before it was dispatched, so it **never prompts a user** — an evidence gap degrades to an `unavailable` block back to TFA, always. @@ -38,7 +38,7 @@ it names no `kubectl` / `chitragupta` / `bifrost`; it routes by *capability*. states the hypothesis and asks TFA to **confirm it against this test's own logs**. - `resume` — optional `{ threadId, turnId }` from a prior PENDING run. - `manifest` — the validated capability manifest `{ capability: { available, via } }` - (built once at the `/factory` gate — Part A). + (built once at the `/rca-build` gate — Part A). If `testRunId` is missing or not parseable as an integer, emit a `failed` `RCA_OUTPUT` block with `root_cause: "no testRunId provided"` and stop — do not diff --git a/automation/README.md b/automation/README.md index d58f5bd..a18e4d4 100644 --- a/automation/README.md +++ b/automation/README.md @@ -31,7 +31,7 @@ PROJECT_NAME="RCA Feature Fencing" BUILD_NAME="VRT Build" \ The upload response carries the build id. Feed it to the plugin: ``` -/factory <build-id> +/rca-build <build-id> ``` ## The failures (what RCA should rediscover) diff --git a/config/rca.config.json b/config/rca.config.json index 021a72c..71e68a3 100644 --- a/config/rca.config.json +++ b/config/rca.config.json @@ -1,5 +1,5 @@ { - "$comment": "Central config for the /factory RCA harness. All formerly-hardcoded product/infra values live here. No kubectl/chitragupta/bifrost literals — connectors are discovered and probe-validated at the gate into the capability manifest (see skills/factory/references/evidence-routing.md). No reportFile: the plugin never writes a local RCA report — the full report lives on the Test Observability UI (triggerRcaReport).", + "$comment": "Central config for the /rca-build RCA harness. All formerly-hardcoded product/infra values live here. No kubectl/chitragupta/bifrost literals — connectors are discovered and probe-validated at the gate into the capability manifest (see skills/rca-build/references/evidence-routing.md). No reportFile: the plugin never writes a local RCA report — the full report lives on the Test Observability UI (triggerRcaReport).", "mcpServerName": "bstack", "concurrency": 5, "turnCap": 6, diff --git a/lib/glimpse.mjs b/lib/glimpse.mjs index 63e328d..c17f8f3 100644 --- a/lib/glimpse.mjs +++ b/lib/glimpse.mjs @@ -1,4 +1,4 @@ -// Terse end-of-run glimpse — the ONLY in-client output of a /factory run. +// Terse end-of-run glimpse — the ONLY in-client output of a /rca-build run. // One line per test: testRunId → cluster → status → confidence one-liner. // The full RCA report lives on the Test Observability UI (triggerRcaReport → // viewReport link); this is deliberately a glimpse, never a report. Degrade, diff --git a/lib/loop.mjs b/lib/loop.mjs index b03ce61..53e83da 100644 --- a/lib/loop.mjs +++ b/lib/loop.mjs @@ -9,7 +9,7 @@ // Pure + dependency-light (imports only the routing registry). // // The loop is fully autonomous: an evidence gap ALWAYS degrades to an -// `unavailable` block back to TFA. There is no user-prompt path — the /factory +// `unavailable` block back to TFA. There is no user-prompt path — the /rca-build // gate closed before this loop ever runs. // // tfaRcaTurn returns TRIMMED terminal shapes: diff --git a/skills/factory/SKILL.md b/skills/rca-build/SKILL.md similarity index 99% rename from skills/factory/SKILL.md rename to skills/rca-build/SKILL.md index f2dc52e..4a4c715 100644 --- a/skills/factory/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -1,9 +1,9 @@ --- -name: factory +name: rca-build description: Single-gate autonomous batch RCA over every failed test of a BrowserStack build via tfaRcaTurn. One gate (connector validation + assumed intake), then fully autonomous — clusters failures, routes evidence, triggers the dashboard report. Args: build id, optional PR URLs / repo hints. --- -# factory — single-gate autonomous RCA over a build +# rca-build — single-gate autonomous RCA over a build Drives the `tfaRcaTurn` collaborative loop over **every failed test** of a build and lands a per-test RCA in the TRA (Test Observability) dashboard. **TFA owns diff --git a/skills/factory/references/clustering.md b/skills/rca-build/references/clustering.md similarity index 100% rename from skills/factory/references/clustering.md rename to skills/rca-build/references/clustering.md diff --git a/skills/factory/references/evidence-routing.md b/skills/rca-build/references/evidence-routing.md similarity index 100% rename from skills/factory/references/evidence-routing.md rename to skills/rca-build/references/evidence-routing.md diff --git a/skills/factory/references/github-evidence.md b/skills/rca-build/references/github-evidence.md similarity index 100% rename from skills/factory/references/github-evidence.md rename to skills/rca-build/references/github-evidence.md diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs index 1fe8699..553b94a 100644 --- a/workflows/rca-batch.mjs +++ b/workflows/rca-batch.mjs @@ -8,7 +8,7 @@ export const meta = { ], }; -// The /factory batch orchestration (fully autonomous — the gate closed before +// The /rca-build batch orchestration (fully autonomous — the gate closed before // this runs; nothing here ever asks the user). This is a dynamic-workflow // script: it runs in the Workflow sandbox (no filesystem, no Date.now/ // Math.random, agent()/pipeline() as globals). It therefore does NO state I/O From e4428c7bfbf04fd9d910d5ab8b54674509f6968c Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 13:18:59 +0530 Subject: [PATCH 24/33] =?UTF-8?q?feat(rca):=20per-build=20state=20file=20i?= =?UTF-8?q?n=20OS=20temp=20=E2=80=94=20csvPathFor(buildId,=20stateDir)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes to the state spine's location: 1. The build id is now IN the filename (rca-state.<buildId>.csv) — previously a fixed .rca/rca-state.csv meant a run on build B would collide with and 'resume' into build A's state. 2. Default directory is OS temp (<tmpdir>/bstack-rca/), not the invoking workspace — a background harness must not pollute the repo it runs from. paths.stateDir remains an override for retained artifacts (e.g. CI). Resume-safety is preserved per build (same id -> same path). Hostile ids are sanitized. 4 new tests; 53/53 green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- config/rca.config.json | 4 ++-- lib/csv-state.mjs | 19 ++++++++++++++++++- skills/rca-build/SKILL.md | 10 ++++++++-- tests/csv-state.test.mjs | 21 +++++++++++++++++++++ 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/config/rca.config.json b/config/rca.config.json index 71e68a3..1928ea1 100644 --- a/config/rca.config.json +++ b/config/rca.config.json @@ -8,8 +8,8 @@ "reaperHeartbeatTtlSec": 600, "errorSummaryMaxChars": 200, "paths": { - "stateDir": ".rca", - "csvFile": ".rca/rca-state.csv" + "$comment": "State CSV path is ALWAYS derived per build via lib/csv-state.mjs csvPathFor(buildId, stateDir): the build id is in the filename (no cross-build collisions) and the default directory is OS temp (<tmpdir>/bstack-rca/), so the harness never pollutes the invoking workspace. Set stateDir only to retain the CSV somewhere specific (e.g. a CI artifact dir).", + "stateDir": "" }, "evidenceRouting": { "test_logs": { "owner": "tfa", "skip": true }, diff --git a/lib/csv-state.mjs b/lib/csv-state.mjs index ea830ff..152e51b 100644 --- a/lib/csv-state.mjs +++ b/lib/csv-state.mjs @@ -14,7 +14,24 @@ // locking is out of scope). import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs"; -import { dirname } from "node:path"; +import { dirname, join } from "node:path"; +import { tmpdir } from "node:os"; + +/** + * Canonical state-file path for one build's run. Two invariants (D-temp): + * 1. The BUILD ID IS IN THE FILENAME — runs over different builds can never + * collide/"resume" into each other's state. + * 2. Default location is OS TEMP (`<tmpdir>/bstack-rca/`), not the invoking + * workspace — a background harness must not pollute the repo it runs from. + * `stateDir` (config `paths.stateDir`) overrides the directory only — e.g. a CI + * job that wants the CSV as a retained artifact. Resume-safety is per build: + * same buildId → same path. + */ +export function csvPathFor(buildId, stateDir = "") { + const safe = String(buildId ?? "").replace(/[^A-Za-z0-9._-]/g, "_") || "unknown-build"; + const dir = stateDir && String(stateDir).trim() !== "" ? String(stateDir) : join(tmpdir(), "bstack-rca"); + return join(dir, `rca-state.${safe}.csv`); +} export const COLUMNS = [ "buildId", diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index 4a4c715..c6e0880 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -109,11 +109,17 @@ listTestIds(buildId=<id>, status="failed", includeFailureDetail=true) (`failure.{category, error_summary, file_path, …}`) — the seed for clustering, so no per-test probe turns are needed. +Resolve the state file with `lib/csv-state.mjs` → `csvPathFor(buildId, +config.paths.stateDir)` — the **build id is in the filename** and the default +directory is **OS temp** (`<tmpdir>/bstack-rca/rca-state.<buildId>.csv`), so +different builds can never collide and the invoking workspace stays clean. Pass +this exact path to the fan-out workflow as `csvPath`. + Seed the CSV/WAL spine from the payload (`lib/csv-state.mjs` → `seed`): one row per failed test, every row `rca_done=pending`, signature columns populated. Re-running `seed` on an existing CSV is idempotent and preserves terminal rows -(resume-safe). If `listTestIds` returns empty → write an empty CSV, report "no -failed tests", stop. +(resume-safe — same build id → same path). If `listTestIds` returns empty → +write an empty CSV, report "no failed tests", stop. ## Step 3 — failure-signature clustering (see references/clustering.md) diff --git a/tests/csv-state.test.mjs b/tests/csv-state.test.mjs index ca55d84..3cbc83a 100644 --- a/tests/csv-state.test.mjs +++ b/tests/csv-state.test.mjs @@ -4,6 +4,7 @@ import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { + csvPathFor, seed, readRows, claim, @@ -164,3 +165,23 @@ test("CSV codec round-trips fields with commas, quotes, newlines", () => { const row = readRows(csv).find((r) => r.testRunId === "200"); assert.equal(row.root_cause, 'Failed: "x", got <y>\nsecond line'); }); + +test("csvPathFor: build id is in the filename, default dir is OS temp", () => { + const p = csvPathFor("abc123XYZ"); + assert.ok(p.startsWith(join(tmpdir(), "bstack-rca"))); + assert.ok(p.endsWith("rca-state.abc123XYZ.csv")); +}); + +test("csvPathFor: different builds never share a path", () => { + assert.notEqual(csvPathFor("build-A"), csvPathFor("build-B")); +}); + +test("csvPathFor: sanitizes hostile ids and handles empty", () => { + assert.ok(csvPathFor("../../etc/passwd").endsWith("rca-state..._.._etc_passwd.csv")); + assert.ok(csvPathFor("").endsWith("rca-state.unknown-build.csv")); +}); + +test("csvPathFor: stateDir override wins over temp", () => { + const p = csvPathFor("b1", "/ci/artifacts"); + assert.equal(p, join("/ci/artifacts", "rca-state.b1.csv")); +}); From 5f345632499d1fb563b006d31e03428a42bb70b8 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 13:31:30 +0530 Subject: [PATCH 25/33] =?UTF-8?q?refactor(rca):=20align=20skills=20with=20?= =?UTF-8?q?canonical=20layout=20=E2=80=94=20templates/,=20examples/,=20scr?= =?UTF-8?q?ipts/?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - k8s-rengg-tfa is self-contained: bin/k8s-context.sh -> skills/k8s-rengg-tfa/ scripts/ (carries the pending redaction hardening for auth headers) - rca-build gains templates/ (suspect-packet, evidence-block, gate-summary) as the canonical fillable formats — references now point at them instead of embedding drift-prone copies — and examples/sample-run.md (fictional, matches the recorded-turn fixtures): gate summary, answered NEEDS_INFO turn with supported + ruled-out suspects, terminal glimpse output Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- README.md | 2 +- skills/rca-build/SKILL.md | 4 +- skills/rca-build/examples/sample-run.md | 77 +++++++++++++++++++ .../rca-build/references/evidence-routing.md | 13 +--- .../rca-build/references/github-evidence.md | 18 ++--- skills/rca-build/templates/evidence-block.md | 26 +++++++ skills/rca-build/templates/gate-summary.md | 21 +++++ skills/rca-build/templates/suspect-packet.md | 22 ++++++ 8 files changed, 160 insertions(+), 23 deletions(-) create mode 100644 skills/rca-build/examples/sample-run.md create mode 100644 skills/rca-build/templates/evidence-block.md create mode 100644 skills/rca-build/templates/gate-summary.md create mode 100644 skills/rca-build/templates/suspect-packet.md diff --git a/README.md b/README.md index 9a3e170..0e51c94 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ A seeded failing build exercises the full loop against real staging infra: `is_mcp_driven` feature and `upload.sh` to push them. The current seeded build: `awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2` (project "RCA Feature Fencing", build "VRT Build"). See `automation/README.md`. -2. **k8s evidence harness** — `skills/k8s-rengg-tfa/` + `bin/k8s-context.sh` +2. **k8s evidence harness** — `skills/k8s-rengg-tfa/` (self-contained: SKILL.md + `scripts/k8s-context.sh`) provide the `k8s` capability: read-only obs-api context (pod health, deployed image, error logs, events) from the `rengg-tfa` namespace, secrets redacted. 3. **Run** — with `BROWSERSTACK_USERNAME`/`ACCESS_KEY` exported and `kubectl` diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index c6e0880..b1839f2 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -80,7 +80,9 @@ is the point: - the current branch for the working branch, - cheap inference (e.g. the automation repo is the cwd if it holds the tests). -Record each assumption in the gate summary ("assumed product repo = +Record each assumption in the gate summary (format: +`templates/gate-summary.md`; worked example: `examples/sample-run.md`) +("assumed product repo = `org/obs-api` from git remote"). A field that cannot be assumed is recorded as "none" and the run proceeds RCA-only for it — **unless** it is both genuinely non-assumable AND load-bearing (in practice: only the build id, and rarely an diff --git a/skills/rca-build/examples/sample-run.md b/skills/rca-build/examples/sample-run.md new file mode 100644 index 0000000..dd5398f --- /dev/null +++ b/skills/rca-build/examples/sample-run.md @@ -0,0 +1,77 @@ +# Example — one full run (fictional data, matches the recorded-turn fixtures) + +Invocation: `/rca-build awswx…fw2` (build id given; nothing else passed). + +## 1. Gate closes (the only user-visible checkpoint) + +``` +GATE CLOSED — capability manifest: + github ✅ valid (gh, authed) · k8s ✅ valid (ctx staging-euc1) · logs ❌ absent · metrics ❌ absent + +Intake: + build id: awswx…fw2 (given) + product repo: acme/obs-api (assumed — from git remote) + automation repo: acme/obs-e2e (assumed — cwd holds the tests) + working branch: main (assumed — current branch) + default branch: main (assumed — origin HEAD) + PRs in play: none (gap) + +Gaps declared to TFA: logs, metrics +Proceeding autonomously: discovery → clustering → fan-out (concurrency 5, turn-cap 6). +``` + +## 2. A NEEDS_INFO turn answered (what the coordinator sends back) + +TFA asked: *"Did request-validation on POST /builds change since last green?"* +(`evidenceType: product_code`, priority high). The coordinator replies on the +same `threadId`: + +``` +ASK: Did request-validation on POST /builds change since last green? +TYPE: product_code +FOUND: yes +SUMMARY: Yes — the buildName validator was tightened to reject empty strings in the +suspect window. One PR touches the failing path; falsification below. +LINK: https://github.com/acme/obs-api/pull/7421 + +SUSPECT: + pr: #7421 + files: src/validators/build.ts + hunks: `- allowEmpty: true` → `+ allowEmpty: false` (validator schema) + author: jdoe + merged_at: 2026-07-01T09:14Z vs last_green: 2026-07-01T02:10Z vs started_at: 2026-07-01T21:40Z + verdict: supported + link: https://github.com/acme/obs-api/pull/7421 + +SUSPECT: + pr: #7418 + files: src/routes/builds.ts + hunks: logging middleware reorder only + author: asmith + merged_at: 2026-06-30T18:02Z vs last_green: 2026-07-01T02:10Z vs started_at: 2026-07-01T21:40Z + verdict: ruled-out (shipped-after check passed but no-path-overlap — hunks never touch the validator) + link: https://github.com/acme/obs-api/pull/7418 + +ASK: Full run logs for test 39 +TYPE: test_logs +FOUND: no +SUMMARY: out-of-scope — TFA owns test logs; skipped by contract. +``` + +## 3. Terminal output (glimpse only — NO local report) + +``` +RCA batch complete — build awswx…fw2 (7 failed → 3 clusters) + +39 → c1 → RESOLVED (high) PR #7421 tightened buildName validator; related_prs: #7421 +41 → c1 → RESOLVED (high) sibling of 39 (cluster confirm) +57 → c2 → RESOLVED (med) flaky selector wait; test-side +81 → c3 → PENDING (—) soft-pending, resumable (turnId recorded) +… + +Full report on the Test Observability UI: +https://observability.browserstack.com/builds/awswx…fw2 +``` + +State file: `<tmpdir>/bstack-rca/rca-state.awswx…fw2.csv` (resume-safe; re-run +the same build id to pick up the PENDING row). diff --git a/skills/rca-build/references/evidence-routing.md b/skills/rca-build/references/evidence-routing.md index afe71aa..d162096 100644 --- a/skills/rca-build/references/evidence-routing.md +++ b/skills/rca-build/references/evidence-routing.md @@ -75,15 +75,10 @@ tail or full PR diff blows both budgets and degrades TFA's reasoning. Supply the ### Per-ask block shape — `ask → found → snippet/link` -``` -ASK: <verbatim `what` from the TfaAsk, ≤ 120 chars> -TYPE: <evidenceType> -FOUND: <yes | no | partial> -SUMMARY: <1–3 sentences — the finding, in the agent's words. ≤ 400 chars> -SNIPPET: - <the load-bearing excerpt only — see size caps. Omit if a LINK fully carries it.> -LINK: <permalink to the source — PR/commit/log-search/metrics panel/deploy record. Omit if N/A.> -``` +**The canonical fillable format lives in +[`../templates/evidence-block.md`](../templates/evidence-block.md)** (fulfilled +and unfulfillable variants) — copy it, don't retype it. Shape: +`ASK / TYPE / FOUND: yes|no|partial / SUMMARY ≤400 / SNIPPET (caps below) / LINK`. - `SUMMARY` is the answer. `SNIPPET` is the *minimum* evidence backing it. `LINK` lets TFA (or a human) verify without the bytes living in the message. diff --git a/skills/rca-build/references/github-evidence.md b/skills/rca-build/references/github-evidence.md index ae1ddcc..4b2c445 100644 --- a/skills/rca-build/references/github-evidence.md +++ b/skills/rca-build/references/github-evidence.md @@ -75,18 +75,12 @@ survives 1–3 is a real candidate; one that fails any is reported as ruled-out ## The suspect packet (structured, not free text) Each surviving/ruled-out suspect is one structured block so `related_prs` -populates deterministically: - -``` -SUSPECT: - pr: <#number> - files: <changed files overlapping the failing path> - hunks: <the 1-3 load-bearing changed hunks — see digest size caps> - author: <login> - merged_at: <ts> vs last_green: <ts> vs started_at: <ts> - verdict: supported | ruled-out (<reason: no-path-overlap | shipped-after | behind-off-flag | unrelated>) - link: <PR permalink> -``` +populates deterministically. **The canonical fillable format lives in +[`../templates/suspect-packet.md`](../templates/suspect-packet.md)** (fields: +pr, files, hunks, author, merged_at vs last_green vs started_at, verdict with +rule-out reason, link) — copy it, don't retype it. A worked example (supported ++ ruled-out side by side) is in +[`../examples/sample-run.md`](../examples/sample-run.md). Only `verdict: supported` suspects should end up in TFA's `related_prs`. Ruled-out suspects stay in the thread as disconfirming evidence so TFA (and a human) can see diff --git a/skills/rca-build/templates/evidence-block.md b/skills/rca-build/templates/evidence-block.md new file mode 100644 index 0000000..fa0bf49 --- /dev/null +++ b/skills/rca-build/templates/evidence-block.md @@ -0,0 +1,26 @@ +# Template — evidence block (one per fulfilled/unfulfilled ask) + +The unit of evidence sent back to TFA in a turn message. Rules, size caps and +the forbidden list: `../references/evidence-routing.md`. + +Fulfilled ask: + +``` +ASK: <verbatim `what` from the TfaAsk, ≤ 120 chars> +TYPE: <evidenceType> +FOUND: <yes | no | partial> +SUMMARY: <1–3 sentences — the finding, in the agent's words. ≤ 400 chars> +SNIPPET: + <the load-bearing excerpt only — see size caps. Omit if a LINK fully carries it.> +LINK: <permalink to the source — PR/commit/log-search/metrics panel/deploy record. Omit if N/A.> +``` + +Unfulfillable ask (report, don't drop — machine-generated for absent connectors +by `lib/loop.mjs` `unavailableBlock`): + +``` +ASK: <verbatim what> +TYPE: <evidenceType> +FOUND: no +SUMMARY: not-found | unreachable | unavailable | out-of-scope — <one line: what was checked or why blocked> +``` diff --git a/skills/rca-build/templates/gate-summary.md b/skills/rca-build/templates/gate-summary.md new file mode 100644 index 0000000..7602968 --- /dev/null +++ b/skills/rca-build/templates/gate-summary.md @@ -0,0 +1,21 @@ +# Template — gate summary (printed once, when THE GATE closes) + +One terse FYI before autonomous execution starts. Every intake field is tagged +`given | assumed | gap`; every connector `valid | invalid | absent`. After this +prints, the run never asks the user anything. + +``` +GATE CLOSED — capability manifest: + github ✅ valid (gh, authed) · k8s ✅ valid (ctx <context>) · logs ❌ absent · metrics ❌ absent + +Intake: + build id: <id> (given) + product repo: <org/repo> (assumed — from git remote) + automation repo: <org/repo> (assumed — cwd holds the tests) + working branch: <branch> (assumed — current branch) + default branch: <branch> (assumed — origin HEAD) + PRs in play: <#123, #456 | none> (given | gap) + +Gaps declared to TFA: <logs, metrics | none> +Proceeding autonomously: discovery → clustering → fan-out (concurrency <N>, turn-cap <M>). +``` diff --git a/skills/rca-build/templates/suspect-packet.md b/skills/rca-build/templates/suspect-packet.md new file mode 100644 index 0000000..676fcf4 --- /dev/null +++ b/skills/rca-build/templates/suspect-packet.md @@ -0,0 +1,22 @@ +# Template — SUSPECT packet (one block per candidate PR) + +Fill one block per suspect, supported **and** ruled-out (elimination is evidence +too). Only `verdict: supported` suspects may feed `related_prs`. Guidance + +falsification protocol: `../references/github-evidence.md`. + +``` +SUSPECT: + pr: <#number> + files: <changed files overlapping the failing path> + hunks: <the 1-3 load-bearing changed hunks — see digest size caps> + author: <login> + merged_at: <ts> vs last_green: <ts> vs started_at: <ts> + verdict: supported | ruled-out (<no-path-overlap | shipped-after | behind-off-flag | unrelated>) + link: <PR permalink> +``` + +If the hunt ends empty after a real search (never fabricate): + +``` +no culprit PR identified after <what was searched: window, repos, paths> +``` From 6796327ff040710edf25b495de7928faaf104312 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 13:36:09 +0530 Subject: [PATCH 26/33] =?UTF-8?q?fix(rca):=20confirmed=20TRA=20UI=20deep-l?= =?UTF-8?q?ink=20=E2=80=94=20automation.browserstack.com=20+=20ai=5Freport?= =?UTF-8?q?/aitfa=20tabs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The viewReport/viewRca destination is the build's AI-TFA report: <base>/builds/<buildUuid>?tab=ai_report&subTab=aitfa on automation.browserstack.com (designer-confirmed; was an observability.browserstack.com assumption). Fixture now carries the guidance string the bridge actually emits on RESOLVED turns — closes the fixture-drift review finding. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- skills/rca-build/examples/sample-run.md | 2 +- tests/conformance.test.mjs | 2 +- tests/fixtures/recorded-turns/resolved.json | 12 ++++++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/skills/rca-build/examples/sample-run.md b/skills/rca-build/examples/sample-run.md index dd5398f..e7b02ea 100644 --- a/skills/rca-build/examples/sample-run.md +++ b/skills/rca-build/examples/sample-run.md @@ -70,7 +70,7 @@ RCA batch complete — build awswx…fw2 (7 failed → 3 clusters) … Full report on the Test Observability UI: -https://observability.browserstack.com/builds/awswx…fw2 +https://automation.browserstack.com/builds/awswx…fw2?tab=ai_report&subTab=aitfa ``` State file: `<tmpdir>/bstack-rca/rca-state.awswx…fw2.csv` (resume-safe; re-run diff --git a/tests/conformance.test.mjs b/tests/conformance.test.mjs index 82f7b8c..3dde60b 100644 --- a/tests/conformance.test.mjs +++ b/tests/conformance.test.mjs @@ -37,7 +37,7 @@ test("resolved fixture: NEEDS_INFO → evidence → RESOLVED, trimmed glimpse ca assert.ok(result.root_cause.length <= 220); // glimpse root_cause is trimmed server-side assert.equal(result.failure_type, "product_regression"); assert.deepEqual(result.related_prs, ["#7421"]); - assert.match(result.view_rca, /^https:\/\/observability\.browserstack\.com\/builds\//); + assert.match(result.view_rca, /^https:\/\/automation\.browserstack\.com/); assert.deepEqual(result.asks_fulfilled, ["product_code"]); assert.deepEqual(result.asks_skipped, ["test_logs"]); // TFA-owned, never gathered assert.equal(result.turns_used, 2); diff --git a/tests/fixtures/recorded-turns/resolved.json b/tests/fixtures/recorded-turns/resolved.json index 47b9034..a7964b8 100644 --- a/tests/fixtures/recorded-turns/resolved.json +++ b/tests/fixtures/recorded-turns/resolved.json @@ -6,7 +6,9 @@ "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-39", - "questions": ["Did the buildName validator change?"], + "questions": [ + "Did the buildName validator change?" + ], "asks": [ { "what": "Did request-validation on POST /builds change since last green?", @@ -29,9 +31,11 @@ "glimpse": { "root_cause": "PR #7421 tightened the buildName validator to reject empty strings", "failure_type": "product_regression", - "related_prs": ["#7421"] + "related_prs": [ + "#7421" + ] }, - "viewRca": "https://observability.browserstack.com/builds/awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2" + "viewRca": "https://automation.browserstack.com — open the build's AI report (tab=ai_report, subTab=aitfa) to view the full RCA" } ] -} +} \ No newline at end of file From a8c201410594dc0dd8dc10cffc0df02dca1cce96 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 13:54:14 +0530 Subject: [PATCH 27/33] chore(rca): remove the k8s-rengg-tfa skill Environment-specific harness removed; k8s stays a generic gate-probed capability (discoveryHints now empty, like logs/metrics). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- README.md | 3 +-- config/rca.config.json | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0e51c94..1344978 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,6 @@ A seeded failing build exercises the full loop against real staging infra: `is_mcp_driven` feature and `upload.sh` to push them. The current seeded build: `awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2` (project "RCA Feature Fencing", build "VRT Build"). See `automation/README.md`. -2. **k8s evidence harness** — `skills/k8s-rengg-tfa/` (self-contained: SKILL.md + `scripts/k8s-context.sh`) provide the `k8s` capability: read-only obs-api context (pod health, deployed image, error logs, events) from the `rengg-tfa` namespace, secrets redacted. 3. **Run** — with `BROWSERSTACK_USERNAME`/`ACCESS_KEY` exported and `kubectl` @@ -109,7 +108,7 @@ A seeded failing build exercises the full loop against real staging infra: ``` /rca-build awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2 ``` - The gate validates connectors (github via `gh`, k8s via the `k8s-rengg-tfa` + The gate validates connectors (github via `gh`, k8s via any k8s-capable skill), then the harness clusters the failures, drives `tfaRcaTurn` per cluster, routes `k8s` asks to the skill while `product_code`/`deploy` asks go to GitHub — landing per-test RCAs on the dashboard that trace back to the diff --git a/config/rca.config.json b/config/rca.config.json index 1928ea1..128500a 100644 --- a/config/rca.config.json +++ b/config/rca.config.json @@ -16,7 +16,7 @@ "product_code": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, "deploy": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, "ci": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, - "k8s": { "capability": "k8s", "discoveryHints": ["k8s-rengg-tfa"] }, + "k8s": { "capability": "k8s", "discoveryHints": [] }, "kibana": { "capability": "logs", "discoveryHints": [] }, "metrics": { "capability": "metrics", "discoveryHints": [] }, "other": { "capability": "other", "discoveryHints": [] } From 838014617db473d304f8a72c316592fd900b0052 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 14:12:30 +0530 Subject: [PATCH 28/33] =?UTF-8?q?refactor(rca):=20generalize=20k8s=20?= =?UTF-8?q?=E2=86=92=20infra=20capability=20=E2=80=94=20route=20to=20whate?= =?UTF-8?q?ver=20runtime=20the=20user=20has?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The harness no longer assumes Kubernetes. 'infra' is the capability (runtime state of the service); k8s/EKS, ECS, docker, Nomad, VMs, PM2 are all just possible connectors behind it — the gate probes what actually exists and records the kind in the manifest (via). The k8s evidenceType remains a routing alias into infra so TFA asks spelled 'k8s' keep working. kubectl is one probe on a menu, not the definition of the capability. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- README.md | 12 ++--- agents/ai-tfa-coordinator.md | 2 +- config/rca.config.json | 53 ++++++++++++++++--- lib/coverage.mjs | 2 +- lib/routing.mjs | 2 +- skills/rca-build/SKILL.md | 9 ++-- skills/rca-build/examples/sample-run.md | 2 +- .../rca-build/references/evidence-routing.md | 9 ++-- skills/rca-build/templates/gate-summary.md | 2 +- tests/coverage-glimpse.test.mjs | 2 +- tests/evidence.test.mjs | 7 +-- tests/routing.test.mjs | 7 +-- 12 files changed, 75 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 1344978..571e7da 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ RCA in the TRA (Test Observability) dashboard. > **The full RCA report lives on the Test Observability UI, not in Claude.** > The plugin surfaces a terse glimpse, triggers the dashboard report > (`triggerRcaReport`), and prints the link. It **discovers and delegates** to -> the infra skills/tools already in your client (GitHub, k8s/EKS, kibana/other +> the infra skills/tools already in your client (GitHub, whatever runtime you have — k8s/ECS/docker/… — kibana/other > logs, metrics). It does **not** install or own those connectors, and it never > writes a local report file. @@ -56,8 +56,8 @@ Args: a build id (bare, `build_id=`, or a dashboard link) plus optional PR URLs The run has exactly **one gate** before execution, with two parts: 1. **Connector discovery + validation** — every connector relevant to test RCA - (github, k8s, logs, metrics, …) is enumerated and probe-validated (`gh auth - status`, `kubectl` reachability, MCP tools listed). The result is a validated + (github, infra, logs, metrics, …) is enumerated and probe-validated (`gh auth + status`, an infra probe matching whatever runtime exists — kubectl/docker/ecs/… — MCP tools listed). The result is a validated capability manifest: `connector → valid | invalid | absent`. A gap is recorded and declared to the TFA agent ("I don't have logs/metrics access") — never a blocker. @@ -101,16 +101,16 @@ A seeded failing build exercises the full loop against real staging infra: `is_mcp_driven` feature and `upload.sh` to push them. The current seeded build: `awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2` (project "RCA Feature Fencing", build "VRT Build"). See `automation/README.md`. - provide the `k8s` capability: read-only obs-api context (pod health, deployed + provide the `infra` capability: read-only runtime context (pod/instance health, deployed image, error logs, events) from the `rengg-tfa` namespace, secrets redacted. 3. **Run** — with `BROWSERSTACK_USERNAME`/`ACCESS_KEY` exported and `kubectl` pointed at the staging cluster: ``` /rca-build awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2 ``` - The gate validates connectors (github via `gh`, k8s via any k8s-capable + The gate validates connectors (github via `gh`, infra via whatever runtime connector exists (kubectl/docker/ecs/…) skill), then the harness clusters the failures, drives `tfaRcaTurn` per - cluster, routes `k8s` asks to the skill while `product_code`/`deploy` asks go + cluster, routes `infra`/`k8s` asks to it while `product_code`/`deploy` asks go to GitHub — landing per-test RCAs on the dashboard that trace back to the seeded regressions, then prints the glimpse + the Test Observability link. diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 49ec5d4..5ef4e88 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -1,6 +1,6 @@ --- name: ai-tfa-coordinator -description: 'Per-test collaborative-RCA coordinator (autonomous — never prompts a user). Given ONE testRunId, drives the tfaRcaTurn MCP loop to a terminal root cause: TFA reads the run logs; this coordinator supplies every non-log evidence ask (product code, k8s, kibana, metrics, deploy, ci) using whatever skills/tools the client has, routed through the validated capability manifest. Skips every test_logs ask (TFA owns logs). For application bugs it MUST hunt the culprit PR via the github connector. Emits a structured RCA_OUTPUT block. Generic over product and infra — no hardcoded tools. Examples: +description: 'Per-test collaborative-RCA coordinator (autonomous — never prompts a user). Given ONE testRunId, drives the tfaRcaTurn MCP loop to a terminal root cause: TFA reads the run logs; this coordinator supplies every non-log evidence ask (product code, infra/runtime, logs, metrics, deploy, ci) using whatever skills/tools the client has, routed through the validated capability manifest. Skips every test_logs ask (TFA owns logs). For application bugs it MUST hunt the culprit PR via the github connector. Emits a structured RCA_OUTPUT block. Generic over product and infra — no hardcoded tools. Examples: - orchestrator: Agent(subagent_type="tfa-rca:ai-tfa-coordinator", prompt="RCA testRunId=39 — error: empty buildName rejected on POST /builds") → drives the loop, returns RCA_OUTPUT - sibling confirm: Agent(subagent_type="tfa-rca:ai-tfa-coordinator", prompt="RCA testRunId=40 — pre-seed: cause=<rep root cause>, suspect PR=#7421") → one-turn confirm against this test logs - user: "run collaborative RCA on test run 39" → single-test loop to RESOLVED/PENDING' diff --git a/config/rca.config.json b/config/rca.config.json index 128500a..732c9e3 100644 --- a/config/rca.config.json +++ b/config/rca.config.json @@ -12,13 +12,50 @@ "stateDir": "" }, "evidenceRouting": { - "test_logs": { "owner": "tfa", "skip": true }, - "product_code": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, - "deploy": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, - "ci": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, - "k8s": { "capability": "k8s", "discoveryHints": [] }, - "kibana": { "capability": "logs", "discoveryHints": [] }, - "metrics": { "capability": "metrics", "discoveryHints": [] }, - "other": { "capability": "other", "discoveryHints": [] } + "test_logs": { + "owner": "tfa", + "skip": true + }, + "product_code": { + "capability": "github", + "discoveryHints": [ + "github-mcp", + "gh" + ] + }, + "deploy": { + "capability": "github", + "discoveryHints": [ + "github-mcp", + "gh" + ] + }, + "ci": { + "capability": "github", + "discoveryHints": [ + "github-mcp", + "gh" + ] + }, + "infra": { + "capability": "infra", + "discoveryHints": [] + }, + "k8s": { + "capability": "infra", + "discoveryHints": [] + }, + "kibana": { + "capability": "logs", + "discoveryHints": [] + }, + "metrics": { + "capability": "metrics", + "discoveryHints": [] + }, + "other": { + "capability": "other", + "discoveryHints": [] + } } } diff --git a/lib/coverage.mjs b/lib/coverage.mjs index caff2d2..0266ceb 100644 --- a/lib/coverage.mjs +++ b/lib/coverage.mjs @@ -1,6 +1,6 @@ // Evidence-coverage stamp (ideation #6, v1 — the per-row coverage band; the // build-level blast-radius digest is deferred). A RESOLVED RCA built with -// k8s+kibana+metrics all "unavailable" must not read like one with full +// infra+logs+metrics all "unavailable" must not read like one with full // evidence. The client (which routed every ask) stamps each row with a coverage // vector and derives a coverage-capped confidence band the reviewer sees: // "low confidence BECAUSE kibana was unavailable", not "low confidence, trust me". diff --git a/lib/routing.mjs b/lib/routing.mjs index 019a012..ba3d699 100644 --- a/lib/routing.mjs +++ b/lib/routing.mjs @@ -99,7 +99,7 @@ export function buildManifest(config, discovered = []) { } // Capabilities that will be unavailable this run — declared to the user up front -// ("k8s + metrics not available") and to TFA so it plans asks around them. +// ("infra + metrics not available") and to TFA so it plans asks around them. export function unavailableCapabilities(manifest) { return Object.entries(manifest) .filter(([, v]) => !v.available) diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index b1839f2..1b7b898 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -7,7 +7,7 @@ description: Single-gate autonomous batch RCA over every failed test of a Browse Drives the `tfaRcaTurn` collaborative loop over **every failed test** of a build and lands a per-test RCA in the TRA (Test Observability) dashboard. **TFA owns -logs; the client agent owns everything else** (product code, k8s, kibana, +logs; the client agent owns everything else** (product code, infra/runtime, logs, metrics, deploy, ci) — routed by capability, generic over product and infra. This skill is the **build-level orchestrator** (`ai-tfa-orchestrator` role). It @@ -47,8 +47,9 @@ pass. The gate has two parts; both run before any RCA work starts. Enumerate every connector relevant to test RCA: - from `config/rca.config.json` → `evidenceRouting`: **github** - (product_code/deploy/ci), **k8s**, **logs** (e.g. kibana), **metrics**, - **other**; + (product_code/deploy/ci), **infra** (whatever runtime the user has — k8s, + ECS, docker, Nomad, plain VMs, PM2, …), **logs** (kibana or any log store), + **metrics**, **other**; - plus any connector-shaped skills / MCP servers present in the session (a log-search MCP, a metrics MCP, an infra skill, …). @@ -57,7 +58,7 @@ Enumerate every connector relevant to test RCA: | Connector | Probe | |---|---| | github | `gh auth status` (or a GitHub MCP tool listed) | -| k8s | k8s skill present **and** `kubectl` reachable (e.g. `kubectl version --request-timeout=5s`) | +| infra | ANY runtime connector the user has — probe what exists, never assume one: `kubectl version --request-timeout=5s`, `docker ps`, `aws ecs list-clusters`, `nomad status`, `pm2 ls`, or an infra-shaped skill/MCP tool. Record the KIND in the manifest (`via: kubectl \| docker \| ecs \| …`) | | logs | a log-search skill/MCP tool actually listed in the session | | metrics | a metrics skill/MCP tool actually listed in the session | | other | best-effort; default `absent` | diff --git a/skills/rca-build/examples/sample-run.md b/skills/rca-build/examples/sample-run.md index e7b02ea..01cc36e 100644 --- a/skills/rca-build/examples/sample-run.md +++ b/skills/rca-build/examples/sample-run.md @@ -6,7 +6,7 @@ Invocation: `/rca-build awswx…fw2` (build id given; nothing else passed). ``` GATE CLOSED — capability manifest: - github ✅ valid (gh, authed) · k8s ✅ valid (ctx staging-euc1) · logs ❌ absent · metrics ❌ absent + github ✅ valid (gh, authed) · infra ✅ valid (via kubectl, ctx staging-euc1) · logs ❌ absent · metrics ❌ absent Intake: build id: awswx…fw2 (given) diff --git a/skills/rca-build/references/evidence-routing.md b/skills/rca-build/references/evidence-routing.md index d162096..05b4435 100644 --- a/skills/rca-build/references/evidence-routing.md +++ b/skills/rca-build/references/evidence-routing.md @@ -42,7 +42,8 @@ An ask that cannot be fulfilled is **never silently dropped** — it becomes a ## Routing table (capability, not tool) `evidenceType` literals are exactly those `tfaRcaTurn` emits: `test_logs`, -`product_code`, `k8s`, `kibana`, `metrics`, `deploy`, `ci`, `other`. +`product_code`, `infra` (TFA may still spell it `k8s` — both route the same), +`kibana`, `metrics`, `deploy`, `ci`, `other`. | `evidenceType` | Capability | Gathered via (discovered at runtime) | |---|---|---| @@ -50,7 +51,7 @@ An ask that cannot be fulfilled is **never silently dropped** — it becomes a | `product_code` | `github` | the client's GitHub capability — **GitHub MCP if present, else `gh`** (see `references/github-evidence.md`) | | `deploy` | `github` | deploy timeline via the GitHub capability (releases/tags + deploy record) | | `ci` | `github` | CI config + run history via the GitHub capability | -| `k8s` | `k8s` | whatever k8s/EKS skill the client has — discovered, not assumed | +| `infra` / `k8s` | `infra` | **whatever runtime connector the user has** — k8s/EKS, ECS, docker, Nomad, plain VMs, PM2, … Discovered and probed at the gate, NEVER assumed to be Kubernetes; the manifest records the kind (`via`) | | `kibana` | `logs` | whatever log-search skill the client has (kibana or other) | | `metrics` | `metrics` | whatever metrics skill the client has | | `other` | `other` | best-effort by ask text; else a `not-found` block | @@ -137,11 +138,11 @@ test, Gate Part A enumerates **and probe-validates** the client's connectors (a recorded gap): ``` -{ github: {available: true, via: "gh"}, k8s: {available: false}, ... } +{ github: {available: true, via: "gh"}, infra: {available: true, via: "kubectl"}, logs: {available: false}, ... } ``` - Every ask routes against this manifest — reproducible, no per-ask discovery. -- The gate summary **declares the gaps to the user** ("k8s + metrics not +- The gate summary **declares the gaps to the user** ("infra + metrics not available") and the first turn declares them to TFA so it plans asks around what's obtainable. - Frozen at gate close. A skill appearing mid-run is not picked up until the next run. diff --git a/skills/rca-build/templates/gate-summary.md b/skills/rca-build/templates/gate-summary.md index 7602968..48e9d94 100644 --- a/skills/rca-build/templates/gate-summary.md +++ b/skills/rca-build/templates/gate-summary.md @@ -6,7 +6,7 @@ prints, the run never asks the user anything. ``` GATE CLOSED — capability manifest: - github ✅ valid (gh, authed) · k8s ✅ valid (ctx <context>) · logs ❌ absent · metrics ❌ absent + github ✅ valid (gh, authed) · infra ✅ valid (via <kubectl ctx …, docker, ecs, …>) · logs ❌ absent · metrics ❌ absent Intake: build id: <id> (given) diff --git a/tests/coverage-glimpse.test.mjs b/tests/coverage-glimpse.test.mjs index 18f2817..4740301 100644 --- a/tests/coverage-glimpse.test.mjs +++ b/tests/coverage-glimpse.test.mjs @@ -29,7 +29,7 @@ test("partial coverage caps a high TFA confidence at medium", () => { test("thin coverage (nothing fulfilled, gaps) caps at low", () => { const s = coverageStamp({ asksFulfilled: [], - asksUnavailable: ["k8s", "metrics"], + asksUnavailable: ["infra", "metrics"], tfaConfidence: "high", }); assert.equal(s.coverage, "thin"); diff --git a/tests/evidence.test.mjs b/tests/evidence.test.mjs index e01b06a..041c694 100644 --- a/tests/evidence.test.mjs +++ b/tests/evidence.test.mjs @@ -8,7 +8,8 @@ const CONFIG = { test_logs: { owner: "tfa", skip: true }, product_code: { capability: "github" }, deploy: { capability: "github" }, - k8s: { capability: "k8s" }, + infra: { capability: "infra" }, + k8s: { capability: "infra" }, metrics: { capability: "metrics" }, other: { capability: "other" }, }, @@ -20,7 +21,7 @@ test("buildManifest marks discovered capabilities available with via", () => { ]); assert.equal(manifest.github.available, true); assert.equal(manifest.github.via, "github-mcp"); - assert.equal(manifest.k8s.available, false); + assert.equal(manifest.infra.available, false); }); test("buildManifest excludes the TFA-owned test_logs capability", () => { @@ -38,7 +39,7 @@ test("buildManifest dedupes capabilities shared by multiple evidence types", () test("unavailableCapabilities lists what the client can't get", () => { const manifest = buildManifest(CONFIG, [{ capability: "github" }]); const unavailable = unavailableCapabilities(manifest).sort(); - assert.deepEqual(unavailable, ["k8s", "metrics", "other"]); + assert.deepEqual(unavailable, ["infra", "metrics", "other"]); }); test("evidence cache computes once and reuses across calls", async () => { diff --git a/tests/routing.test.mjs b/tests/routing.test.mjs index c63b595..6ebdb9b 100644 --- a/tests/routing.test.mjs +++ b/tests/routing.test.mjs @@ -6,7 +6,8 @@ const CONFIG = { evidenceRouting: { test_logs: { owner: "tfa", skip: true }, product_code: { capability: "github", discoveryHints: ["github-mcp", "gh"] }, - k8s: { capability: "k8s", discoveryHints: [] }, + infra: { capability: "infra", discoveryHints: [] }, + k8s: { capability: "infra", discoveryHints: [] }, other: { capability: "other", discoveryHints: [] }, }, }; @@ -30,10 +31,10 @@ test("available capability → gather, carrying via", () => { test("unavailable capability → gap, carrying discovery hints", () => { const r = routeAsk({ evidenceType: "k8s", priority: "medium" }, CONFIG, { - k8s: { available: false }, + infra: { available: false }, }); assert.equal(r.action, "gap"); - assert.equal(r.capability, "k8s"); + assert.equal(r.capability, "infra"); assert.equal(r.reason, "no-capability"); }); From af0f188715387c829807209ad4c41c8d535926bf Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 15:40:47 +0530 Subject: [PATCH 29/33] docs(rca): fix orphaned demo-step lines left by the k8s-skill removal; note prod default base URL Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 571e7da..17c23a0 100644 --- a/README.md +++ b/README.md @@ -101,10 +101,9 @@ A seeded failing build exercises the full loop against real staging infra: `is_mcp_driven` feature and `upload.sh` to push them. The current seeded build: `awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2` (project "RCA Feature Fencing", build "VRT Build"). See `automation/README.md`. - provide the `infra` capability: read-only runtime context (pod/instance health, deployed - image, error logs, events) from the `rengg-tfa` namespace, secrets redacted. -3. **Run** — with `BROWSERSTACK_USERNAME`/`ACCESS_KEY` exported and `kubectl` - pointed at the staging cluster: +2. **Run** — with `BROWSERSTACK_USERNAME`/`ACCESS_KEY` exported (and, for a + staging build like this one, `O11Y_TFA_RCA_BASE_URL` pointed at the tenant — + the default is production; see `INTEGRATION.md`): ``` /rca-build awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2 ``` From 37edaa8cb1e3d7bf59808c147db7fc559cef6718 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 15:52:44 +0530 Subject: [PATCH 30/33] chore(rca): pin bstack MCP server to @browserstack/mcp-server@1.2.27-beta.1 Pinned across all three client wirings (.mcp.json, .cursor-mcp.json, codex example) + INTEGRATION.md. Codex example env aligned with the prod-default base URL (staging override optional). Version verified present on the public npm registry. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- .cursor-mcp.json | 2 +- .mcp.json | 2 +- INTEGRATION.md | 4 ++-- codex-mcp.example.toml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.cursor-mcp.json b/.cursor-mcp.json index 9cf95af..eed3690 100644 --- a/.cursor-mcp.json +++ b/.cursor-mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "bstack": { "command": "npx", - "args": ["-y", "@browserstack/mcp-server"], + "args": ["-y", "@browserstack/mcp-server@1.2.27-beta.1"], "env": { "BROWSERSTACK_USERNAME": "${BROWSERSTACK_USERNAME}", "BROWSERSTACK_ACCESS_KEY": "${BROWSERSTACK_ACCESS_KEY}", diff --git a/.mcp.json b/.mcp.json index 0502929..7fca468 100644 --- a/.mcp.json +++ b/.mcp.json @@ -3,7 +3,7 @@ "bstack": { "type": "stdio", "command": "npx", - "args": ["-y", "@browserstack/mcp-server"], + "args": ["-y", "@browserstack/mcp-server@1.2.27-beta.1"], "env": { "BROWSERSTACK_USERNAME": "${BROWSERSTACK_USERNAME}", "BROWSERSTACK_ACCESS_KEY": "${BROWSERSTACK_ACCESS_KEY}", diff --git a/INTEGRATION.md b/INTEGRATION.md index 65de0b6..f7bedad 100644 --- a/INTEGRATION.md +++ b/INTEGRATION.md @@ -76,7 +76,7 @@ into `~/.codex/config.toml`, or: codex mcp add bstack \ --env BROWSERSTACK_USERNAME=… --env BROWSERSTACK_ACCESS_KEY=… \ --env O11Y_TFA_RCA_BASE_URL=https://api-observability-rengg-tfa.bsstag.com \ - -- npx -y @browserstack/mcp-server + -- npx -y @browserstack/mcp-server@1.2.27-beta.1 ``` **Skill + agent discovery** — Codex reads `.agents/skills/` (skills) and @@ -92,7 +92,7 @@ Then run the `rca-build` skill; the coordinator + `tfaRcaTurn` loop are identica ## Notes -- The `bstack` server is **stdio** (`npx @browserstack/mcp-server`), not a remote +- The `bstack` server is **stdio** (`npx @browserstack/mcp-server@1.2.27-beta.1`), not a remote OAuth server — so the configs use `command`/`args`/`env`, unlike Slack's `url`+`oauth`/`auth` shape. - Env-var interpolation (`${VAR}`) is honored by Claude Code's `.mcp.json`; on diff --git a/codex-mcp.example.toml b/codex-mcp.example.toml index 4fb24af..e3dda80 100644 --- a/codex-mcp.example.toml +++ b/codex-mcp.example.toml @@ -5,7 +5,7 @@ [mcp_servers.bstack] command = "npx" -args = ["-y", "@browserstack/mcp-server"] -env = { "BROWSERSTACK_USERNAME" = "your-username", "BROWSERSTACK_ACCESS_KEY" = "your-access-key", "O11Y_TFA_RCA_BASE_URL" = "https://api-observability-rengg-tfa.bsstag.com" } +args = ["-y", "@browserstack/mcp-server@1.2.27-beta.1"] +env = { "BROWSERSTACK_USERNAME" = "your-username", "BROWSERSTACK_ACCESS_KEY" = "your-access-key", "O11Y_TFA_RCA_BASE_URL" = "" # optional: set only to target a staging tenant (default is production) } startup_timeout_sec = 15 tool_timeout_sec = 120 From 6fea22854c678dc0f4ae1cd5c8e61c919c800495 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 17:18:12 +0530 Subject: [PATCH 31/33] chore(rca): remove automation/ seed harness + harden product-repo inference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The automation/ demo (failing rcaChat API cases + upload.sh, named observability-api as 'the product repo') was env-specific bait: the gate latched onto the only repo named in a workspace doc and carried its PRs into the manifest — even for a build whose failures were self-healing tests, wholly unrelated to observability-api. - Delete automation/ (tests, upload.sh, README, fixture xml). - Gate Part B now REQUIRES product-repo corroboration against the failure signatures; a doc-sourced repo is a weak hint, never an assumption — if it doesn't match the failures it's recorded as a gap and culprit-PR hunts report 'no culprit PR identified' rather than blaming an unrelated repo. - README 'Demo run' section replaced with a generic 'Run' section (no seeded build, no hardcoded repo). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- README.md | 37 +++++------ automation/README.md | 45 ------------- automation/build-rca-failures.xml | 74 --------------------- automation/tests/test_rca_chat_api.py | 96 --------------------------- automation/upload.sh | 23 ------- skills/rca-build/SKILL.md | 12 ++++ 6 files changed, 30 insertions(+), 257 deletions(-) delete mode 100644 automation/README.md delete mode 100644 automation/build-rca-failures.xml delete mode 100644 automation/tests/test_rca_chat_api.py delete mode 100755 automation/upload.sh diff --git a/README.md b/README.md index 17c23a0..ca64af1 100644 --- a/README.md +++ b/README.md @@ -93,25 +93,24 @@ mandatory culprit-PR links on application bugs — is the real deliverable. skills your client already has. Missing ones degrade gracefully (the RCA's confidence band reflects what evidence was actually available). -## Demo run (rengg-tfa) - -A seeded failing build exercises the full loop against real staging infra: - -1. **Seed the build** — `automation/` holds failing API cases for the rcaChat / - `is_mcp_driven` feature and `upload.sh` to push them. The current seeded build: - `awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2` (project "RCA Feature Fencing", - build "VRT Build"). See `automation/README.md`. -2. **Run** — with `BROWSERSTACK_USERNAME`/`ACCESS_KEY` exported (and, for a - staging build like this one, `O11Y_TFA_RCA_BASE_URL` pointed at the tenant — - the default is production; see `INTEGRATION.md`): - ``` - /rca-build awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2 - ``` - The gate validates connectors (github via `gh`, infra via whatever runtime connector exists (kubectl/docker/ecs/…) - skill), then the harness clusters the failures, drives `tfaRcaTurn` per - cluster, routes `infra`/`k8s` asks to it while `product_code`/`deploy` asks go - to GitHub — landing per-test RCAs on the dashboard that trace back to the - seeded regressions, then prints the glimpse + the Test Observability link. +## Run + +Point it at any red BrowserStack build — the harness discovers what it needs: + +``` +# BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY exported; default base is +# production, override O11Y_TFA_RCA_BASE_URL only for a staging tenant. +/rca-build <build-id> +``` + +The single gate validates connectors (github via `gh`, infra via whatever +runtime connector exists — kubectl/docker/ecs/…) and resolves the intake +**by inference** (product/automation repo from the cwd's git remote, branches, +any PRs you pass). It never assumes a product repo from unrelated workspace +docs — if it can't infer one that matches the failures, it records the gap and +proceeds RCA-only rather than blaming the wrong repo. Then it clusters the +failures, drives `tfaRcaTurn` per cluster, and lands per-test RCAs on the +dashboard, printing the glimpse + the Test Observability link. ## Layout diff --git a/automation/README.md b/automation/README.md deleted file mode 100644 index a18e4d4..0000000 --- a/automation/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# Automation — RCA Feature Fencing - -Failing API automation for the rcaChat / `is_mcp_driven` ownership feature -(observability-api, PRs #8305 / #7114 / #7059), run against the **rengg-tfa** -staging env. The failing build it produces is the **input the RCA plugin runs -against** — each failure pins a real regression so the collaborative RCA has -genuine material to trace back to our commits. - -## Files -- `tests/test_rca_chat_api.py` — the automation cases (pytest + requests). -- `build-rca-failures.xml` — JUnit results (6 failures + 1 error) for upload. -- `upload.sh` — pushes the JUnit XML to Observability, creating the project/build. - -## Credentials -Never commit creds. Export them (or `source automation/.env`, gitignored): -```bash -export BSTACK_USER=tfauser_wzgsM5 -export BSTACK_KEY=<access-key> -export O11Y_BASE_URL=https://api-observability-rengg-tfa.bsstag.com -``` - -## Run + upload -```bash -# (optional) regenerate the XML against the live API -pytest automation/tests --junitxml=automation/build-rca-failures.xml - -# upload → creates project "RCA Feature Fencing", build "VRT Build" -PROJECT_NAME="RCA Feature Fencing" BUILD_NAME="VRT Build" \ - ./automation/upload.sh automation/build-rca-failures.xml -``` - -The upload response carries the build id. Feed it to the plugin: -``` -/rca-build <build-id> -``` - -## The failures (what RCA should rediscover) -| Test | Pins | -|---|---| -| `mcp_claim_refused_when_web_owned` | jsonb `::` cast → SQLState 42601 → 500 (TestRunsRcaRepository) | -| `mcp_claim_sets_is_mcp_driven_true` | jsonb_set never applied (same root cause) | -| `web_approve_after_mcp_claim_is_not_locked_out` | cross-flow ownership lockout | -| `submit_turn_returns_structured_status` | setTestRca 500 before turn produced | -| `needs_info_asks_carry_evidence_type` | ask.evidenceType null | -| `set_test_rca_error_callback_does_not_500` | data-gate not scoped to success (AIService) | diff --git a/automation/build-rca-failures.xml b/automation/build-rca-failures.xml deleted file mode 100644 index ce78926..0000000 --- a/automation/build-rca-failures.xml +++ /dev/null @@ -1,74 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<testsuites name="RCA Feature Fencing" tests="7" failures="6" errors="1" time="38.402"> - <testsuite name="RcaChat Ownership Fencing" tests="4" failures="3" errors="1" time="19.871" timestamp="2026-06-24T09:12:03"> - <testcase classname="com.browserstack.observability.rcachat.OwnershipFencingTest" name="mcp_claim_refused_when_web_owned" time="4.310" file="api-tests/rcachat/test_ownership_fencing.py"> - <failure message="Expected HTTP 403 (FORBIDDEN) when the run is web-owned (metadata.is_mcp_driven=false); got HTTP 500."> -AssertionError: claimForMcpOrRefuse did not refuse a web-owned run. - Request : POST /ext/v1/testRuns/4412/rcaChat {"message":"start"} - Expected: 403 FORBIDDEN (web flow owns this RCA) - Actual : 500 Internal Server Error - Body : {"error":"could not execute statement [n/a]; SQL [n/a]; nested exception is - org.postgresql.util.PSQLException: ERROR: syntax error at or near \":\" - Position: 71 ; SQLState: 42601"} - Hint : native UPDATE in TestRunsRcaRepository.claimForMcpIfNotWebOwned uses '::jsonb' - which Hibernate parses as a named parameter. - at api-tests/rcachat/test_ownership_fencing.py:48 - </failure> - </testcase> - <testcase classname="com.browserstack.observability.rcachat.OwnershipFencingTest" name="mcp_claim_sets_is_mcp_driven_true_when_unowned" time="3.901" file="api-tests/rcachat/test_ownership_fencing.py"> - <failure message="Expected metadata.is_mcp_driven=true after an MCP claim on an unowned run; got null."> -AssertionError: claim did not stamp ownership. - Request : POST /ext/v1/testRuns/4413/rcaChat - Expected: test_runs_rca.metadata.is_mcp_driven == true - Actual : metadata.is_mcp_driven == null (jsonb_set never applied — statement failed) - at api-tests/rcachat/test_ownership_fencing.py:71 - </failure> - </testcase> - <testcase classname="com.browserstack.observability.rcachat.OwnershipFencingTest" name="web_approve_after_mcp_claim_is_not_locked_out" time="5.220" file="api-tests/rcachat/test_ownership_fencing.py"> - <failure message="Cross-flow lockout: web approve() left the run MCP-owned."> -AssertionError: expected web approve() to win the cross-flow race; run stayed is_mcp_driven=true. - Sequence: MCP claim -> web approve() - Expected: is_mcp_driven=false after approve - Actual : is_mcp_driven=true - at api-tests/rcachat/test_ownership_fencing.py:96 - </failure> - </testcase> - <testcase classname="com.browserstack.observability.rcachat.OwnershipFencingTest" name="seed_pending_row_when_no_row_exists" time="6.440" file="api-tests/rcachat/test_ownership_fencing.py"> - <error message="Connection reset while seeding PENDING row"> -java.net.SocketException: Connection reset - at com.browserstack.observability.service.RcaChatService.claimForMcpOrRefuse(RcaChatService.java) - while seeding a PENDING test_runs_rca row for testRunId=4414 - </error> - </testcase> - </testsuite> - <testsuite name="RcaChat Turn API" tests="3" failures="3" errors="0" time="18.531" timestamp="2026-06-24T09:12:24"> - <testcase classname="com.browserstack.observability.rcachat.TurnApiTest" name="submit_turn_returns_structured_status" time="6.110" file="api-tests/rcachat/test_turn_api.py"> - <failure message="Expected a structured turn (status in NEEDS_INFO|RESOLVED|BLOCKED|PENDING); got 500."> -AssertionError: rcaChat submit did not return a structured turn. - Request : POST /ext/v1/testRuns/4420/rcaChat {"message":"empty buildName rejected on POST /builds"} - Expected: 200 with body.status in [NEEDS_INFO, RESOLVED, BLOCKED, PENDING] - Actual : 500 Internal Server Error (setTestRca failed before turn was produced) - at api-tests/rcachat/test_turn_api.py:39 - </failure> - </testcase> - <testcase classname="com.browserstack.observability.rcachat.TurnApiTest" name="needs_info_asks_carry_evidence_type" time="5.980" file="api-tests/rcachat/test_turn_api.py"> - <failure message="Each ask in a NEEDS_INFO turn must carry an evidenceType; one ask had null."> -AssertionError: ask.evidenceType missing. - Turn : NEEDS_INFO with 3 asks - Expected: every ask.evidenceType in [test_logs, product_code, k8s, kibana, metrics, deploy, ci, other] - Actual : asks[2].evidenceType == null - at api-tests/rcachat/test_turn_api.py:64 - </failure> - </testcase> - <testcase classname="com.browserstack.observability.rcachat.TurnApiTest" name="set_test_rca_error_callback_does_not_500" time="6.441" file="api-tests/rcachat/test_turn_api.py"> - <failure message="Non-chat error callback (success=false, no data) returned 500 instead of 200."> -AssertionError: setTestRca data-gate is not scoped to success. - Request : error callback {"success":false} (no rca data) - Expected: 200 (record error state, no RCA) - Actual : 500 Internal Server Error (gate rejected missing data even on the error path) - Hint : AIService.setTestRca data-gate must be scoped to success=true. - at api-tests/rcachat/test_turn_api.py:88 - </failure> - </testcase> - </testsuite> -</testsuites> diff --git a/automation/tests/test_rca_chat_api.py b/automation/tests/test_rca_chat_api.py deleted file mode 100644 index 2f84af2..0000000 --- a/automation/tests/test_rca_chat_api.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Failing API automation for the rcaChat / is_mcp_driven ownership feature. - -Targets the observability-api rcaChat surface deployed in the rengg-tfa staging -env. These cases are written to FAIL against the current build — each one pins a -real regression in the feature we shipped (PRs #8305 / #7114 / #7059): - - - the jsonb '::' cast in TestRunsRcaRepository.claimForMcpIfNotWebOwned - (Hibernate parses '::jsonb' as a named param -> SQLState 42601 -> 500) - - AIService.setTestRca data-gate not scoped to success (error callbacks 500) - - cross-flow ownership lockout between MCP claim and web approve() - -Run with: pytest --junitxml=automation/build-rca-failures.xml -Creds + base URL come from the environment (never hardcode): - O11Y_BASE_URL default https://api-observability-rengg-tfa.bsstag.com - BSTACK_USER / BSTACK_KEY -""" - -import os - -import pytest -import requests - -BASE = os.environ.get("O11Y_BASE_URL", "https://api-observability-rengg-tfa.bsstag.com") -AUTH = (os.environ.get("BSTACK_USER", ""), os.environ.get("BSTACK_KEY", "")) -TIMEOUT = 30 - - -def _rca_chat(test_run_id, body): - return requests.post( - f"{BASE}/ext/v1/testRuns/{test_run_id}/rcaChat", - json=body, - auth=AUTH, - timeout=TIMEOUT, - ) - - -class TestOwnershipFencing: - def test_mcp_claim_refused_when_web_owned(self): - # A web-owned run (metadata.is_mcp_driven=false) must refuse the MCP claim - # with 403 — not 500 from a broken native UPDATE. - resp = _rca_chat(4412, {"message": "start"}) - assert resp.status_code == 403, ( - f"expected 403 FORBIDDEN for a web-owned run, got {resp.status_code}: {resp.text}" - ) - - def test_mcp_claim_sets_is_mcp_driven_true_when_unowned(self): - resp = _rca_chat(4413, {"message": "start"}) - assert resp.status_code == 200 - # ownership should be stamped on the row - meta = resp.json().get("metadata", {}) - assert meta.get("is_mcp_driven") is True, "claim did not stamp is_mcp_driven=true" - - def test_web_approve_after_mcp_claim_is_not_locked_out(self): - _rca_chat(4414, {"message": "start"}) # MCP claim - approve = requests.post( - f"{BASE}/ext/v1/testRuns/4414/rca/approve", auth=AUTH, timeout=TIMEOUT - ) - assert approve.status_code == 200 - state = requests.get( - f"{BASE}/ext/v1/testRuns/4414/rca", auth=AUTH, timeout=TIMEOUT - ).json() - assert state.get("metadata", {}).get("is_mcp_driven") is False, ( - "cross-flow lockout: web approve() left the run MCP-owned" - ) - - -class TestTurnApi: - def test_submit_turn_returns_structured_status(self): - resp = _rca_chat(4420, {"message": "empty buildName rejected on POST /builds"}) - assert resp.status_code == 200, f"expected a structured turn, got {resp.status_code}" - assert resp.json().get("status") in { - "NEEDS_INFO", - "RESOLVED", - "BLOCKED", - "PENDING", - } - - def test_needs_info_asks_carry_evidence_type(self): - resp = _rca_chat(4421, {"message": "investigate"}) - asks = resp.json().get("asks", []) - valid = {"test_logs", "product_code", "k8s", "kibana", "metrics", "deploy", "ci", "other"} - for i, ask in enumerate(asks): - assert ask.get("evidenceType") in valid, f"asks[{i}].evidenceType missing/invalid" - - def test_set_test_rca_error_callback_does_not_500(self): - # Error callback (success=false, no rca data) must record the error state - # and return 200 — the data-gate must be scoped to success=true. - resp = requests.post( - f"{BASE}/ext/v1/testRuns/4422/rca/callback", - json={"success": False}, - auth=AUTH, - timeout=TIMEOUT, - ) - assert resp.status_code == 200, ( - f"error callback returned {resp.status_code}; data-gate not scoped to success" - ) diff --git a/automation/upload.sh b/automation/upload.sh deleted file mode 100755 index cd4f3bf..0000000 --- a/automation/upload.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# Upload the JUnit results to BrowserStack Observability (rengg-tfa staging), -# creating the project + build that the RCA plugin then runs against. -# -# Creds come from the environment — never commit them: -# export BSTACK_USER=... BSTACK_KEY=... -# (or `source automation/.env`, which is gitignored.) -set -euo pipefail - -UPLOAD_URL="${O11Y_UPLOAD_URL:-https://upload-observability-rengg-tfa.bsstag.com/upload}" -XML="${1:-$(dirname "$0")/build-rca-failures.xml}" -PROJECT="${PROJECT_NAME:-RCA Feature Fencing}" -BUILD="${BUILD_NAME:-VRT Build}" - -: "${BSTACK_USER:?set BSTACK_USER}" -: "${BSTACK_KEY:?set BSTACK_KEY}" - -curl -sS -X POST "$UPLOAD_URL" \ - -u "${BSTACK_USER}:${BSTACK_KEY}" \ - -F "data=@${XML}" \ - -F "projectName=${PROJECT}" \ - -F "buildName=${BUILD}" -echo diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index 1b7b898..86ee86a 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -81,6 +81,18 @@ is the point: - the current branch for the working branch, - cheap inference (e.g. the automation repo is the cwd if it holds the tests). +**Product-repo corroboration (do NOT skip).** The product repo must plausibly +be the *system under test for THIS build's failures* — not merely a repo name +found lying around. A repo mentioned only in workspace docs/READMEs is a **weak +hint, never an assumption**: cross-check it against the failure signatures +(discovery runs first if needed) — do the failing area, files, or error strings +relate to that repo's domain? If they don't (e.g. the failures are self-healing +`healedElement is null` cases but the only named repo is an observability API), +the product repo is a **gap**, recorded as "unknown" — the run proceeds RCA-only +and every culprit-PR hunt reports "no culprit PR identified" rather than blaming +an unrelated repo's PRs. Never carry a doc-sourced repo (or its PRs) into the +manifest as a settled product repo without this corroboration. + Record each assumption in the gate summary (format: `templates/gate-summary.md`; worked example: `examples/sample-run.md`) ("assumed product repo = From c68cd17248f5c8cf7d326d145b0e1694ae554afc Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 17:42:34 +0530 Subject: [PATCH 32/33] fix(rca): ask for the product repo when it's unknown + load-bearing (interactive) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously the gate silently recorded 'product repo: gap' whenever corroboration failed — which kills the mandatory culprit-PR hunt without telling the human sitting at the terminal. Now, when the repo can't be corroborated against the failure signatures AND no PRs were supplied, the product repo is treated as non-assumable AND load-bearing and earns the single consolidated gate question (interactive only): 'Failures look like <domain>; which repo owns that code? (reply none → RCA without culprit-PR attribution).' Headless still records the gap and proceeds RCA-only. Preserves both invariants: never blame the wrong repo, and never silently cripple culprit-PR hunting when a human could answer. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- skills/rca-build/SKILL.md | 27 +++++++++++++++------- skills/rca-build/templates/gate-summary.md | 6 ++++- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index 86ee86a..26450a7 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -88,20 +88,31 @@ hint, never an assumption**: cross-check it against the failure signatures (discovery runs first if needed) — do the failing area, files, or error strings relate to that repo's domain? If they don't (e.g. the failures are self-healing `healedElement is null` cases but the only named repo is an observability API), -the product repo is a **gap**, recorded as "unknown" — the run proceeds RCA-only -and every culprit-PR hunt reports "no culprit PR identified" rather than blaming -an unrelated repo's PRs. Never carry a doc-sourced repo (or its PRs) into the -manifest as a settled product repo without this corroboration. +the doc-sourced repo is discarded — never carry it (or its PRs) into the +manifest as a settled product repo. + +When corroboration leaves **no** product repo, decide by whether a human can help: +- **PRs were supplied** → treat those as the suspect surface; product repo is + derived from them. No question needed. +- **No PRs, interactive session** → the product repo is now **non-assumable AND + load-bearing** (without it the mandatory culprit-PR hunt is dead), so it earns + the single consolidated gate question below — ask it; don't silently degrade. +- **No PRs, headless** → record the gap ("product repo: unknown") and proceed + RCA-only; every culprit-PR hunt reports "no culprit PR identified". Record each assumption in the gate summary (format: `templates/gate-summary.md`; worked example: `examples/sample-run.md`) ("assumed product repo = `org/obs-api` from git remote"). A field that cannot be assumed is recorded as "none" and the run proceeds RCA-only for it — **unless** it is both genuinely -non-assumable AND load-bearing (in practice: only the build id, and rarely an -ambiguous repo when PRs were supplied). Those, and only those, may be asked -**ONCE, in a single consolidated question at gate close**. Never a second -question. **Headless: skip asking entirely; record the gaps.** +non-assumable AND load-bearing. In practice that set is: the build id; **the +product repo when it could not be corroborated and no PRs were supplied** (see +above — without it the culprit-PR hunt cannot run); and rarely an ambiguous repo +when PRs were supplied. Those, and only those, may be asked **ONCE, in a single +consolidated question at gate close** — e.g. *"Failures look like `<domain>`; +which repo owns that code? (reply 'none' → I'll RCA without culprit-PR +attribution)."* Never a second question. **Headless: skip asking entirely; +record the gaps.** ### Gate close diff --git a/skills/rca-build/templates/gate-summary.md b/skills/rca-build/templates/gate-summary.md index 48e9d94..48ac6bf 100644 --- a/skills/rca-build/templates/gate-summary.md +++ b/skills/rca-build/templates/gate-summary.md @@ -4,13 +4,17 @@ One terse FYI before autonomous execution starts. Every intake field is tagged `given | assumed | gap`; every connector `valid | invalid | absent`. After this prints, the run never asks the user anything. +If the product repo could not be corroborated against the failures AND no PRs +were supplied, the gate asks ONE question (interactive only) before printing +this summary. Headless skips it and prints `product repo: unknown (gap)`. + ``` GATE CLOSED — capability manifest: github ✅ valid (gh, authed) · infra ✅ valid (via <kubectl ctx …, docker, ecs, …>) · logs ❌ absent · metrics ❌ absent Intake: build id: <id> (given) - product repo: <org/repo> (assumed — from git remote) + product repo: <org/repo> (assumed — corroborated vs failures | asked — human answered | unknown (gap)) automation repo: <org/repo> (assumed — cwd holds the tests) working branch: <branch> (assumed — current branch) default branch: <branch> (assumed — origin HEAD) From 31915f0f8ae433f033036f29686a4d8f79c84f6f Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 21:00:27 +0530 Subject: [PATCH 33/33] =?UTF-8?q?fix(rca):=20finish=20output=20is=20a=20co?= =?UTF-8?q?mpletion=20notice=20+=20link=20only=20=E2=80=94=20no=20RCA=20de?= =?UTF-8?q?tail=20in=20Claude?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The end-of-run glimpse was leaking exactly what's supposed to live only on the TRA UI: per-test root causes, culprit/related PRs, cluster breakdowns, confidence rationales. renderGlimpse now emits just 'RCA analysis complete — build <id>' + a status-count line (N tests · R resolved · P pending · F failed); the caller adds the 'Full report on the Test Observability UI: <link>' line. Step 6 and the example updated to match; a test asserts no testRunId/cluster/root_cause/PR ever appears in the output. Humans open the link for the why. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- lib/glimpse.mjs | 69 +++++++++++-------------- skills/rca-build/SKILL.md | 26 +++++----- skills/rca-build/examples/sample-run.md | 14 ++--- tests/coverage-glimpse.test.mjs | 32 +++++------- 4 files changed, 65 insertions(+), 76 deletions(-) diff --git a/lib/glimpse.mjs b/lib/glimpse.mjs index c17f8f3..38d7fb0 100644 --- a/lib/glimpse.mjs +++ b/lib/glimpse.mjs @@ -1,50 +1,41 @@ -// Terse end-of-run glimpse — the ONLY in-client output of a /rca-build run. -// One line per test: testRunId → cluster → status → confidence one-liner. -// The full RCA report lives on the Test Observability UI (triggerRcaReport → -// viewReport link); this is deliberately a glimpse, never a report. Degrade, -// don't crash: missing fields render as "-". +// Terse end-of-run summary — the ONLY in-client output of a /rca-build run. +// Deliberately a COMPLETION NOTICE, not a report: status counts + the UI link, +// nothing else. Root causes, culprit PRs, per-test analysis, cluster breakdowns +// live ONLY on the Test Observability dashboard (triggerRcaReport → viewReport). +// Do not reintroduce per-test cause/PR lines here — that is the whole point of +// the trim. Degrade, don't crash: missing fields are treated as absent. import { readRows } from "./csv-state.mjs"; -const DASH = "-"; -const ONE_LINER_MAX = 80; - -function cell(value) { - const s = value == null ? "" : String(value).trim(); - if (s === "") return DASH; - return s.replace(/\s*\n\s*/g, " "); -} - -function oneLiner(row) { - const confidence = cell(row.confidence); - const cause = cell(row.root_cause); - const text = cause === DASH ? confidence : `${confidence}: ${cause}`; - return text.length > ONE_LINER_MAX ? `${text.slice(0, ONE_LINER_MAX - 1)}…` : text; +// Map raw CSV rca_done states → the three buckets a human cares about. +function bucket(state) { + const s = (state || "").toLowerCase(); + if (s === "resolved") return "resolved"; + if (s === "pending" || s === "pending-resume") return "pending"; + return "failed"; } -// Render the glimpse from CSV rows. Returns a plain-text block: -// <testRunId> → <cluster_id> → <status> → <confidence one-liner> +// Render the completion summary. Returns a plain-text block: +// RCA analysis complete — build <id> +// <N> tests · <R> resolved · <P> pending · <F> failed +// (the caller appends the "Full report on the Test Observability UI: <link>" +// line from triggerRcaReport's viewReport — see SKILL.md Step 6). export function renderGlimpse(rows, { buildId } = {}) { - const lines = [`Glimpse${buildId ? ` — build ${buildId}` : ""}`]; + const head = `RCA analysis complete${buildId ? ` — build ${buildId}` : ""}`; if (!rows || rows.length === 0) { - lines.push("No failed tests analyzed."); - return lines.join("\n") + "\n"; - } - const byState = rows.reduce((acc, r) => { - const k = r.rca_done || "unknown"; - acc[k] = (acc[k] ?? 0) + 1; - return acc; - }, {}); - const summary = Object.entries(byState) - .map(([k, v]) => `${k}: ${v}`) - .join(" · "); - lines.push(`${rows.length} test(s) — ${summary}`); - for (const r of rows) { - lines.push( - `${cell(r.testRunId)} → ${cell(r.cluster_id)} → ${cell(r.rca_done)} → ${oneLiner(r)}`, - ); + return `${head}\nNo failed tests analyzed.\n`; } - return lines.join("\n") + "\n"; + const counts = rows.reduce( + (acc, r) => { + acc[bucket(r.rca_done)] += 1; + return acc; + }, + { resolved: 0, pending: 0, failed: 0 }, + ); + const parts = [`${rows.length} test(s)`, `${counts.resolved} resolved`]; + if (counts.pending) parts.push(`${counts.pending} pending`); + if (counts.failed) parts.push(`${counts.failed} failed`); + return `${head}\n${parts.join(" · ")}\n`; } export function renderGlimpseFromCsv(csvPath, opts = {}) { diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index 26450a7..140b2a9 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -193,24 +193,26 @@ identified after <what was searched>" and the CSV row records the gap. ## Step 6 — finish: glimpse + dashboard report (NO local report) -This plugin **never renders or writes a local RCA report**. When every row is -terminal: - -1. Print a **terse glimpse table** from the CSV (`lib/glimpse.mjs` → - `renderGlimpse`): one line per test — `testRunId → cluster → status → - confidence one-liner`. That is the entire in-Claude output. -2. Call the MCP tool **`triggerRcaReport(buildUuid=<build id>)`** (add - `force=true` only to re-run over an existing completed report). It returns a - trimmed glimpse (`state, verdict, verdictProvisional, partial, analyzedCount, - totalFailedCount, totalPrs, faultyPrNumbers, failureReason, viewReport`). +This plugin **never renders or writes a local RCA report, and never surfaces RCA +detail in Claude.** The in-Claude output is a two-line completion notice plus the +link — that is all. When every row is terminal: + +1. Print the **completion summary** from the CSV (`lib/glimpse.mjs` → + `renderGlimpse`): `RCA analysis complete — build <id>` + a status count line + (`<N> tests · <R> resolved · <P> pending · <F> failed`). **Nothing per-test.** +2. Call **`triggerRcaReport(buildUuid=<build id>)`** (add `force=true` only to + re-run over an existing completed report). 3. Print the link line, verbatim shape: ``` Full report on the Test Observability UI: <viewReport> ``` -Humans read the real report **there**, populated by the BrowserStack agent — -not in Claude. +**Do NOT print** root causes, culprit/related PRs, cluster breakdowns, per-test +analysis, confidence rationales, or a per-test table — root_cause, related_prs, +suspect_signals and the like are for the CSV + the dashboard ONLY. If a human +wants the "why", they open the link. Claude's job here is "analysis complete → +report is at <link>", not to re-narrate the RCA the BrowserStack agent authored. ## Resume diff --git a/skills/rca-build/examples/sample-run.md b/skills/rca-build/examples/sample-run.md index 01cc36e..5efa6e2 100644 --- a/skills/rca-build/examples/sample-run.md +++ b/skills/rca-build/examples/sample-run.md @@ -61,17 +61,17 @@ SUMMARY: out-of-scope — TFA owns test logs; skipped by contract. ## 3. Terminal output (glimpse only — NO local report) ``` -RCA batch complete — build awswx…fw2 (7 failed → 3 clusters) - -39 → c1 → RESOLVED (high) PR #7421 tightened buildName validator; related_prs: #7421 -41 → c1 → RESOLVED (high) sibling of 39 (cluster confirm) -57 → c2 → RESOLVED (med) flaky selector wait; test-side -81 → c3 → PENDING (—) soft-pending, resumable (turnId recorded) -… +RCA analysis complete — build awswx…fw2 +7 test(s) · 6 resolved · 1 pending Full report on the Test Observability UI: https://automation.browserstack.com/builds/awswx…fw2?tab=ai_report&subTab=aitfa ``` +That is the **entire** in-Claude output. No root causes, no culprit PRs, no +per-test table — those are on the dashboard, authored by the BrowserStack agent. +The RESOLVED turn shown earlier is the coordinator↔TFA exchange (internal to the +loop), not something re-printed to the user at the end. + State file: `<tmpdir>/bstack-rca/rca-state.awswx…fw2.csv` (resume-safe; re-run the same build id to pick up the PENDING row). diff --git a/tests/coverage-glimpse.test.mjs b/tests/coverage-glimpse.test.mjs index 4740301..3bc0e48 100644 --- a/tests/coverage-glimpse.test.mjs +++ b/tests/coverage-glimpse.test.mjs @@ -54,7 +54,7 @@ test("empty batch renders a valid glimpse, no crash", () => { assert.match(txt, /No failed tests analyzed/); }); -test("glimpse renders one arrow line per test with status counts", () => { +test("glimpse is a completion notice with counts — NO per-test detail", () => { const rows = [ { testRunId: "101", @@ -62,24 +62,22 @@ test("glimpse renders one arrow line per test with status counts", () => { rca_done: "resolved", confidence: "high", root_cause: "PR #7421 tightened validator", + related_prs: "#7421", }, - { - testRunId: "102", - cluster_id: "c1", - rca_done: "failed", - confidence: "", - root_cause: "", - }, + { testRunId: "102", cluster_id: "c1", rca_done: "pending-resume" }, + { testRunId: "103", cluster_id: "c2", rca_done: "failed" }, ]; const txt = renderGlimpse(rows, { buildId: "b1" }); - assert.match(txt, /2 test\(s\)/); - assert.match(txt, /resolved: 1/); - assert.match(txt, /failed: 1/); - assert.match(txt, /101 → c1 → resolved → high: PR #7421 tightened validator/); - assert.match(txt, /102 → c1 → failed → -/); // blank fields degrade to "-" + assert.match(txt, /RCA analysis complete — build b1/); + assert.match(txt, /3 test\(s\)/); + assert.match(txt, /1 resolved/); + assert.match(txt, /1 pending/); // pending-resume buckets to "pending" + assert.match(txt, /1 failed/); + // The trim: no per-test lines, no root cause, no PRs, no testRunIds leak. + assert.doesNotMatch(txt, /7421|PR #|→|101|102|103|c1|c2/); }); -test("glimpse one-liner truncates and collapses newlines (stays terse)", () => { +test("glimpse never leaks a verbose root_cause, however long", () => { const rows = [ { testRunId: "1", @@ -90,8 +88,6 @@ test("glimpse one-liner truncates and collapses newlines (stays terse)", () => { }, ]; const txt = renderGlimpse(rows); - const line = txt.split("\n").find((l) => l.startsWith("1 →")); - assert.ok(line.includes("line one line two")); - assert.ok(line.endsWith("…")); - assert.ok(line.length < 120); + assert.doesNotMatch(txt, /line one|line two|xxxx/); // cause stays out of Claude + assert.match(txt, /1 test\(s\) · 1 resolved/); });