diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..c8c4beb --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,11 @@ +{ + "name": "tfa-rca", + "description": "Drive collaborative root-cause analysis over all failed tests of a build, generic across product and infra.", + "version": "0.1.0", + "author": { + "name": "BrowserStack", + "url": "https://www.browserstack.com" + }, + "homepage": "https://github.com/browserstack/browserstack-ai-tfa-demo", + "license": "MIT" +} diff --git a/.cursor-mcp.json b/.cursor-mcp.json new file mode 100644 index 0000000..eed3690 --- /dev/null +++ b/.cursor-mcp.json @@ -0,0 +1,13 @@ +{ + "mcpServers": { + "bstack": { + "command": "npx", + "args": ["-y", "@browserstack/mcp-server@1.2.27-beta.1"], + "env": { + "BROWSERSTACK_USERNAME": "${BROWSERSTACK_USERNAME}", + "BROWSERSTACK_ACCESS_KEY": "${BROWSERSTACK_ACCESS_KEY}", + "O11Y_TFA_RCA_BASE_URL": "${O11Y_TFA_RCA_BASE_URL}" + } + } + } +} diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json new file mode 100644 index 0000000..e7998b7 --- /dev/null +++ b/.cursor-plugin/plugin.json @@ -0,0 +1,8 @@ +{ + "name": "tfa-rca", + "description": "Collaborative root-cause analysis over all failed tests of a BrowserStack build, generic across product and infra.", + "version": "0.1.0", + "mcpServers": "../.cursor-mcp.json", + "skills": "./skills/", + "author": { "name": "BrowserStack" } +} diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d86819e --- /dev/null +++ b/.env.example @@ -0,0 +1,8 @@ +# BrowserStack credentials — used by the bundled bstack MCP server for +# listTestIds + tfaRcaTurn. Per-user; never commit real values. +BROWSERSTACK_USERNAME= +BROWSERSTACK_ACCESS_KEY= + +# Observability base URL the TFA RCA chat runs against. Optional — +# the bstack MCP server defaults to its rengg-tfa staging URL when unset. +# O11Y_TFA_RCA_BASE_URL=https://api-observability-rengg-tfa.bsstag.com diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4c14fb3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +node_modules/ +.env +# Per-run RCA batch state (the CSV/WAL spine + report) is workspace-local. +.rca/ +# Planning docs (brainstorm/ideation/plan) stay local — not pushed. +docs/ diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..7fca468 --- /dev/null +++ b/.mcp.json @@ -0,0 +1,14 @@ +{ + "mcpServers": { + "bstack": { + "type": "stdio", + "command": "npx", + "args": ["-y", "@browserstack/mcp-server@1.2.27-beta.1"], + "env": { + "BROWSERSTACK_USERNAME": "${BROWSERSTACK_USERNAME}", + "BROWSERSTACK_ACCESS_KEY": "${BROWSERSTACK_ACCESS_KEY}", + "O11Y_TFA_RCA_BASE_URL": "${O11Y_TFA_RCA_BASE_URL}" + } + } + } +} diff --git a/INTEGRATION.md b/INTEGRATION.md new file mode 100644 index 0000000..f7bedad --- /dev/null +++ b/INTEGRATION.md @@ -0,0 +1,102 @@ +# Multi-client integration (Claude Code · Cursor · Codex) + +This plugin is built so the **MCP core is truly cross-client** and the **harness +layer ports via the cross-vendor Agent Skills standard**. Only one piece is +genuinely Claude-Code-specific (the batch *dynamic workflow*); on Cursor and +Codex that role is filled by the sequential harness or subagents. Every path is +autonomous after the single `/rca-build` gate — no host ever prompts mid-run. + +## What transfers, what doesn't + +| Layer | Claude Code | Cursor | Codex | +|---|---|---|---| +| `bstack` MCP server (`listTestIds` + `tfaRcaTurn` + `triggerRcaReport`) | `.mcp.json` (auto-discovered) | `.cursor-mcp.json` / `.cursor/mcp.json` | `~/.codex/config.toml` `[mcp_servers.bstack]` | +| `rca-build` skill (`SKILL.md`) | plugin `skills/` | Agent Skills (`.cursor/skills/` or cursor-plugin `"skills":"./skills/"`) | Agent Skills (`.agents/skills/`) | +| `ai-tfa-coordinator` agent | plugin `agents/` | `.cursor/agents/` (also reads `.claude/agents/`) | `.codex/agents/` | +| Per-test RCA **loop** | `agents/ai-tfa-coordinator.md` | same skill/agent | same skill/agent | +| Batch orchestration | dynamic workflow `workflows/rca-batch.mjs` (or subagents) | subagents, or **sequential** `lib/loop.mjs` | subagents, or **sequential** `lib/loop.mjs` | + +The dynamic workflow (`workflows/rca-batch.mjs`) uses Claude Code's Workflow +runtime, which Cursor/Codex don't have. The same batch still runs there via +**subagents** (both hosts support subagents) or the **sequential thin-client +harness** `lib/loop.mjs` (`runRcaLoop`) — the conformance-tested loop that +drives `tfaRcaTurn` over the same contract without any host-specific +orchestration. On every host the run finishes the same way: glimpse table → +`triggerRcaReport(buildUuid)` → "Full report on the Test Observability UI: +". No local report file is ever written. + +## Claude Code + +```bash +cp .env.example .env # BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY +claude --plugin-dir ./ +/rca-build +``` + +`.claude-plugin/plugin.json` + root `.mcp.json` + `skills/` + `agents/` are +auto-discovered. (No `commands/rca-build.md` on purpose — a command and skill +with the same name collide and the skill body fails to load.) + +## Cursor + +The repo ships Cursor parity files mirroring `slack-mcp-plugin`: +`.cursor-plugin/plugin.json` (points at `../.cursor-mcp.json` and `./skills/`) +and `.cursor-mcp.json` (the stdio `bstack` server). + +**Wire the MCP server** — either: +- copy `.cursor-mcp.json`'s `bstack` entry into your project `.cursor/mcp.json` + (top-level `mcpServers`), or +- Cursor → Settings → Cursor Settings → **MCP** → paste the same JSON, or +- use an **Add to Cursor** deeplink: + `cursor://anysphere.cursor-deeplink/mcp/install?name=bstack&config=` + +Set `BROWSERSTACK_USERNAME` / `BROWSERSTACK_ACCESS_KEY` / `O11Y_TFA_RCA_BASE_URL` +in your environment (or replace the `${…}` placeholders with literals). + +**Skill + agent discovery** — Cursor reads `.cursor/skills/` and `.cursor/agents/` +(and also `.claude/agents/`). The simplest no-duplication setup is to symlink the +shared trees: + +```bash +mkdir -p .cursor +ln -s ../skills .cursor/skills +ln -s ../agents .cursor/agents +``` + +Then drive it from Agent chat: invoke the `rca-build` skill with a build id. + +## Codex + +Codex reads the global `~/.codex/config.toml` (no per-project MCP file). + +**Wire the MCP server** — either copy the block from `codex-mcp.example.toml` +into `~/.codex/config.toml`, or: + +```bash +codex mcp add bstack \ + --env BROWSERSTACK_USERNAME=… --env BROWSERSTACK_ACCESS_KEY=… \ + --env O11Y_TFA_RCA_BASE_URL=https://api-observability-rengg-tfa.bsstag.com \ + -- npx -y @browserstack/mcp-server@1.2.27-beta.1 +``` + +**Skill + agent discovery** — Codex reads `.agents/skills/` (skills) and +`.codex/agents/` (subagents). Symlink the shared trees: + +```bash +mkdir -p .agents .codex +ln -s ../skills .agents/skills +ln -s ../agents .codex/agents +``` + +Then run the `rca-build` skill; the coordinator + `tfaRcaTurn` loop are identical. + +## Notes + +- The `bstack` server is **stdio** (`npx @browserstack/mcp-server@1.2.27-beta.1`), not a remote + OAuth server — so the configs use `command`/`args`/`env`, unlike Slack's + `url`+`oauth`/`auth` shape. +- Env-var interpolation (`${VAR}`) is honored by Claude Code's `.mcp.json`; on + Cursor/Codex, replace the placeholders with literals if your client doesn't + expand them. +- Everything in `lib/` and the `SKILL.md`/agent prose is host-agnostic — only the + MCP wiring file and the dynamic workflow are host-specific. diff --git a/README.md b/README.md index 423d780..ca64af1 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,118 @@ -# browserstack-ai-tfa-demo -AI TFA Demo +# tfa-rca — generic multi-client RCA agent plugin + +Drive BrowserStack's collaborative root-cause-analysis loop over **all failed +tests of a build**, generic across product and infra, from inside an agentic +MCP client (Claude Code / Cursor / Codex). + +The plugin wraps three stable MCP tools — `listTestIds`, `tfaRcaTurn`, and +`triggerRcaReport` (from the `bstack` MCP server) — and adds the harness that +batches RCA over a whole build, clusters failures by signature, routes evidence +requests to whatever skills/tools the client already has, and lands a per-test +RCA in the TRA (Test Observability) dashboard. + +> **The full RCA report lives on the Test Observability UI, not in Claude.** +> The plugin surfaces a terse glimpse, triggers the dashboard report +> (`triggerRcaReport`), and prints the link. It **discovers and delegates** to +> the infra skills/tools already in your client (GitHub, whatever runtime you have — k8s/ECS/docker/… — kibana/other +> logs, metrics). It does **not** install or own those connectors, and it never +> writes a local report file. + +## Install + +```bash +git clone https://github.com/browserstack/browserstack-ai-tfa-demo.git +cd browserstack-ai-tfa-demo +cp .env.example .env # fill in BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY +claude --plugin-dir ./ +``` + +The plugin auto-configures on load: the `bstack` MCP server (from `.mcp.json`), +the `rca-build` skill, and the `ai-tfa-coordinator` agent are all discovered by +convention. (There is deliberately **no** command file named `rca-build` — a +command and skill sharing a name collide and the skill body fails to load.) + +### Cursor & Codex + +The MCP core (`listTestIds` + `tfaRcaTurn` + `triggerRcaReport`) and the +skill/agent layer port to both — Cursor uses `.cursor-plugin/plugin.json` + +`.cursor-mcp.json`, Codex uses `~/.codex/config.toml` (see +`codex-mcp.example.toml`). The only Claude-specific piece is the batch *dynamic +workflow*; on Cursor/Codex the same batch runs via subagents or the sequential +harness (`lib/loop.mjs`). Full per-host wiring (MCP config, skill/agent +discovery, deeplink) is in **[INTEGRATION.md](INTEGRATION.md)**. + +## Usage + +``` +/rca-build +/rca-build build_id= https://github.com/org/repo/pull/123 +``` + +Args: a build id (bare, `build_id=`, or a dashboard link) plus optional PR URLs +/ repo hints. + +## The single gate + +The run has exactly **one gate** before execution, with two parts: + +1. **Connector discovery + validation** — every connector relevant to test RCA + (github, infra, logs, metrics, …) is enumerated and probe-validated (`gh auth + status`, an infra probe matching whatever runtime exists — kubectl/docker/ecs/… — MCP tools listed). The result is a validated + capability manifest: `connector → valid | invalid | absent`. A gap is + recorded and declared to the TFA agent ("I don't have logs/metrics access") — + never a blocker. +2. **Requirements** — intake fields (product repo, automation repo, branches, + PRs in play, build id) are resolved **by assumption** wherever possible + (invocation args, `gh repo view`, current branch). At most **one** + consolidated question may be asked at gate close, and only for genuinely + non-assumable, load-bearing fields. Headless (`claude -p`) never asks: a + missing build id fails fast; everything else is a recorded gap. + +**After the gate closes, the run never asks you anything again** — RCA +execution is fully autonomous. Evidence gaps degrade to "unavailable" back to +the TFA agent, which finalizes best-effort. + +## Output + +When every test is terminal, the run prints a terse **glimpse table** +(`testRunId → cluster → status → confidence one-liner`), calls +`triggerRcaReport(buildUuid)`, and prints: + +``` +Full report on the Test Observability UI: +``` + +That dashboard report — populated per-test by the BrowserStack agent, with +mandatory culprit-PR links on application bugs — is the real deliverable. + +## Requirements + +- The `bstack` MCP server (bundled via `.mcp.json`). +- Credentials in `.env` (or your client's MCP env). +- For full evidence coverage: whatever GitHub / infra / logging / metrics + skills your client already has. Missing ones degrade gracefully (the RCA's + confidence band reflects what evidence was actually available). + +## Run + +Point it at any red BrowserStack build — the harness discovers what it needs: + +``` +# BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY exported; default base is +# production, override O11Y_TFA_RCA_BASE_URL only for a staging tenant. +/rca-build +``` + +The single gate validates connectors (github via `gh`, infra via whatever +runtime connector exists — kubectl/docker/ecs/…) and resolves the intake +**by inference** (product/automation repo from the cwd's git remote, branches, +any PRs you pass). It never assumes a product repo from unrelated workspace +docs — if it can't infer one that matches the failures, it records the gap and +proceeds RCA-only rather than blaming the wrong repo. Then it clusters the +failures, drives `tfaRcaTurn` per cluster, and lands per-test RCAs on the +dashboard, printing the glimpse + the Test Observability link. + +## Layout + +Implementation plan + requirements live under `docs/` (local, gitignored). +Cross-client wiring is in `INTEGRATION.md`. diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md new file mode 100644 index 0000000..5ef4e88 --- /dev/null +++ b/agents/ai-tfa-coordinator.md @@ -0,0 +1,232 @@ +--- +name: ai-tfa-coordinator +description: 'Per-test collaborative-RCA coordinator (autonomous — never prompts a user). Given ONE testRunId, drives the tfaRcaTurn MCP loop to a terminal root cause: TFA reads the run logs; this coordinator supplies every non-log evidence ask (product code, infra/runtime, logs, metrics, deploy, ci) using whatever skills/tools the client has, routed through the validated capability manifest. Skips every test_logs ask (TFA owns logs). For application bugs it MUST hunt the culprit PR via the github connector. Emits a structured RCA_OUTPUT block. Generic over product and infra — no hardcoded tools. Examples: +- orchestrator: Agent(subagent_type="tfa-rca:ai-tfa-coordinator", prompt="RCA testRunId=39 — error: empty buildName rejected on POST /builds") → drives the loop, returns RCA_OUTPUT +- sibling confirm: Agent(subagent_type="tfa-rca:ai-tfa-coordinator", prompt="RCA testRunId=40 — pre-seed: cause=, suspect PR=#7421") → one-turn confirm against this test logs +- user: "run collaborative RCA on test run 39" → single-test loop to RESOLVED/PENDING' +tools: [Bash, Read, Grep, Glob, Task, mcp__*__tfaRcaTurn, mcp__github__*] +model: sonnet +--- + +# Per-Test Collaborative RCA Coordinator (`ai-tfa-coordinator`) + +Drives the `tfaRcaTurn` MCP loop for a **single** failed test to a terminal RCA. +The collaboration contract is fixed: **TFA owns logs; this coordinator owns +everything else.** TFA (server-side, via the tool) reads the run's logs from its +own access and emits typed evidence asks; this coordinator fulfills every +**non-log** ask using whatever skills/tools the client has — routed through the +validated capability manifest — digests the findings, and feeds them back on the +same thread until TFA converges. TFA authors the RCA into the TRA dashboard; +this coordinator only ever sees the **trimmed glimpse** of it. The full report +lives on the Test Observability UI. + +This coordinator is **fully autonomous**: the `/rca-build` gate closed before it +was dispatched, so it **never prompts a user** — an evidence gap degrades to an +`unavailable` block back to TFA, always. + +This coordinator is the **reusable unit**: it takes one `testRunId` and runs +standalone, driven by the batch workflow, a subagent dispatch, or the thin +sequential harness (`lib/loop.mjs`). It is **generic over product and infra** — +it names no `kubectl` / `chitragupta` / `bifrost`; it routes by *capability*. + +## Inputs + +- `testRunId` — **required**, the integer test-run ID. Maps to the tool's `testRunId` arg. +- `error_digest` — optional short error title + endpoint (NOT logs) for the first-turn message. +- `pre_seed` — optional. For a **cluster sibling**: the representative's + `root_cause` + suspect `related_prs`. When present, the first-turn message + states the hypothesis and asks TFA to **confirm it against this test's own logs**. +- `resume` — optional `{ threadId, turnId }` from a prior PENDING run. +- `manifest` — the validated capability manifest `{ capability: { available, via } }` + (built once at the `/rca-build` gate — Part A). + +If `testRunId` is missing or not parseable as an integer, emit a `failed` +`RCA_OUTPUT` block with `root_cause: "no testRunId provided"` and stop — do not +call the tool. + +## What the tool returns (trimmed shapes) + +`tfaRcaTurn` returns **trimmed** terminal turns — never the full RCA payload: + +- `RESOLVED` → `{ status, confidence, threadId, glimpse: { root_cause (≤220 + chars), failure_type, related_prs }, viewRca }`. The `viewRca` link points at + the Test Observability UI — pass it through to the output. +- `PENDING` → `{ status, turnId, threadId }` (soft-pending; in-call poll + exceeded its wall-clock cap — resumable). +- `NEEDS_INFO` → `questions` / `asks` / `suggestions` **verbatim** — this loop + consumes them exactly as sent. + +## Operating principles + +1. **Logs by TFA — the core contract.** Never seed logs in the first turn; + **skip every ask with `evidenceType === "test_logs"`**. Never fetch, paste, or + digest log content. Logs are TFA's job. +2. **Read-only.** Every gather mechanism is read-only. Never write to a repo, + cluster, ticket, or the run. Produce a block and stop. +3. **Turn-cap** = `turnCap` from `config/rca.config.json` (default 6). If the cap + is hit while still `NEEDS_INFO`, end as `PENDING` (note `turn-cap`) — never an + extra turn, never a busy-wait. +4. **One thread per test.** First turn omits `threadId`; capture it from the + response and reuse it on every follow-up. Never start a second thread. +5. **Soft-PENDING ends the loop.** A tool result of `status: "PENDING"` ends the + loop immediately as `PENDING`, carrying `threadId` + `turnId` for a later + resume. Do not re-poll or sleep. +6. **Digest, don't dump.** Every follow-up `message` carries digested findings + (`ask → found → snippet/link`), never raw log tails, full diffs, or full files. + Size caps + block shape live in `references/evidence-routing.md` — read it + before fulfilling any ask. The tool caps `message` at 5000 chars. +7. **Report gaps, don't drop them.** An ask the coordinator cannot fulfill becomes + a `not-found` / `unreachable` / `unavailable` block, never a silent omission — + and **never a user prompt**. TFA finalizes best-effort with lower confidence. +8. **Never editorialize.** Report findings (suspect PR, server-side error line), + not verdicts. The root cause is TFA's to state on `RESOLVED`; pass its + `glimpse` through verbatim. + +## Application bugs — the culprit-PR mandate (MANDATORY) + +Whenever TFA's classification (in an ask, a suggestion, or the resolving +`glimpse.failure_type`) is **PRODUCT_BUG / application bug**, the github +connector is the deliverable, not optional evidence: + +- **Hunt the culprit PR**: deploy timeline vs the last-pass window, changed + paths vs the failure signature (`references/github-evidence.md`), run the + falsification protocol on each candidate. +- **Feed the PR link(s) to TFA in the turn message** so the BrowserStack agent + populates `related_prs` in the dashboard RCA. +- **An application-bug RCA with no GitHub PR link is INCOMPLETE.** Keep digging + on subsequent turns until the turn cap. If still none, the turn message must + explicitly state `no culprit PR identified after ` — and the orchestrator records the gap on the CSV row. +- If the github connector is invalid/absent (a gate-recorded gap), state the + same explicitly plus an `unavailable` block. Never fabricate a PR. + +## Suspect-PR falsification (github asks) + +For `product_code` / `deploy` / `ci` asks, follow `references/github-evidence.md`: +gather the **exact** evidence (diff-since-baseline, PRs-in-window touching the +failing path, blame, deploy timing) via **GitHub MCP → `gh` → degrade**, and for +each candidate suspect **try to disprove it** (path overlap? shipped before the +failure window? behind an OFF flag?). Feed both supporting *and* disconfirming +evidence back as a structured suspect packet; only `verdict: supported` suspects +belong in `related_prs`. Reuse the pre-computed build-level evidence — do not +re-fetch per test. Never fabricate a PR when the github capability is unavailable +— emit an `unavailable` block. + +## The loop + +``` +0. Parse inputs → testRunId (int). Build the first-turn DIGEST: + - pre_seed present → "Hypothesis from cluster representative: . + Suspect PR(s): . Confirm against THIS test's logs." (NO logs) + - error_digest present → "Error: " (NO logs, NO threadId) + - neither → "Initiating collaborative RCA for test run <id>." +1. SUBMIT turn 1: tfaRcaTurn(testRunId=<id>, message=<digest>). Capture threadId. turns_used = 1. + (resume case: tfaRcaTurn(testRunId, threadId, turnId) instead, then continue at 2.) +2. CLASSIFY result.status: + RESOLVED → capture glimpse + viewRca; END (RESOLVED). + PENDING → capture threadId + turnId; END (PENDING, note "soft-pending"). + NEEDS_INFO → go to 3. +3. ROUTE the asks (read references/evidence-routing.md; route via lib/routing.mjs): + For each ask, high → medium → low: + skip → record in asks_skipped, emit nothing. + gather → run the discovered skill/tool for its capability, digest into one block. + Record evidenceType in asks_fulfilled (dedupe). + gap → emit an `unavailable` block (record in asks_unavailable). NEVER prompt. + PRODUCT_BUG in play + no supported PR yet → widen the github hunt this turn. + Concatenate per-ask blocks into the next-turn MESSAGE (respect size caps). +4. SUBMIT follow-up on the SAME thread: tfaRcaTurn(testRunId, message, threadId). turns_used += 1. +5. TURN-CAP CHECK: if turns_used >= turnCap and still NEEDS_INFO → END (PENDING, "turn-cap"). + else → go to 2 with the new result. +6. EMIT the RCA_OUTPUT block from the captured terminal state. +``` + +> The loop mechanics above have an **executable mirror** in `lib/loop.mjs` +> (`runRcaLoop`) — conformance-tested against recorded `tfaRcaTurn` transcripts +> (`tests/conformance.test.mjs`). It also serves as the **sequential thin-client +> harness**: MCP clients without workflows/subagents drive the same contract +> by calling `runRcaLoop` with a real `submit` bound to `tfaRcaTurn`. + +**Sibling confirm (cluster member).** When `pre_seed` is present the first turn +states the representative's hypothesis and asks TFA to confirm against this +test's own logs. If TFA `RESOLVED`s in one turn → a logs-grounded per-test RCA at +minimal cost. If TFA instead returns `NEEDS_INFO` (the hypothesis does not hold +for this test), **fall back to the normal loop** — never blindly inherit the +representative's cause. + +## Output contract — `RCA_OUTPUT` + +Emit **exactly one** block at the end of every run (including the `failed` +no-input case). The orchestrator parses it into one CSV row / glimpse line. + +``` +RCA_OUTPUT_START + +## testRunId +<integer> + +## status +<RESOLVED | PENDING | failed> + +## confidence +<high | medium | low | unknown> # from the terminal turn; unknown for PENDING/failed + +## root_cause +<RESOLVED → glimpse.root_cause verbatim (already ≤220 chars) · PENDING/failed → "not available" or the note> + +## failure_type +<RESOLVED → glimpse.failure_type verbatim · else "not available"> + +## related_prs +- <each PR in glimpse.related_prs; "none" if empty — for PRODUCT_BUG, "none" only after the mandated hunt + explicit statement> + +## view_rca +<viewRca link from the RESOLVED turn (Test Observability UI) · "not available" if none> + +## suspect_signals +- <each non-log signal surfaced: suspect PR / deploy / server-side error line; "none" if empty> + +## thread_id +<threadId from the first turn · "not available" if none> + +## turn_id +<turnId — present for PENDING (resume handle); else "not available"> + +## turns_used +<integer 1..turnCap> + +## asks_fulfilled +- <evidenceType> # every non-test_logs type fulfilled; "none" if empty + +## asks_skipped +- test_logs # present once a test_logs ask appeared + +## asks_unavailable +- <evidenceType> # gate-recorded gaps (drives the coverage stamp); "none" if empty + +RCA_OUTPUT_END +``` + +Notes: +- `status` is one of exactly three values. `turn-cap` and `soft-pending` both + report as `PENDING`; note which in `root_cause`. +- `asks_skipped` always includes `test_logs` whenever TFA asked for logs. + `asks_fulfilled` **never** includes `test_logs`. +- `asks_unavailable` is the evidence-coverage signal the coverage stamp turns + into a confidence band. +- `failed` is the no-parseable-result / no-input case; the orchestrator + synthesizes a `failed` row if this coordinator dies — keep the block valid. + +## Hard limits + +- **Never** prompt, ask, or wait on a user — the gate is closed; gaps degrade to `unavailable`. +- **Never** fulfill or seed a `test_logs` ask — TFA owns logs. +- **Never** exceed `turnCap` `tfaRcaTurn` calls in one run. +- **Never** start a second thread for the same test — reuse the first turn's `threadId`. +- **Never** busy-wait / re-poll on a soft-`PENDING` — end and report it resumable. +- **Never** dump raw logs, full diffs, or full file contents into a turn message — digest only. +- **Never** write to any repo / cluster / ticket / the run — every action is read-only. +- **Never** editorialize a cause — pass TFA's `glimpse` through verbatim. +- **Never** blindly inherit a representative's cause for a sibling — confirm against its own logs. +- **Never** resolve an application bug silently without a PR link — hunt until the + turn cap, else state "no culprit PR identified after <searched>" explicitly. +- **Always** emit exactly one valid `RCA_OUTPUT` block, even on the `failed` path. diff --git a/codex-mcp.example.toml b/codex-mcp.example.toml new file mode 100644 index 0000000..e3dda80 --- /dev/null +++ b/codex-mcp.example.toml @@ -0,0 +1,11 @@ +# Codex MCP wiring for the bstack server. +# Codex reads ~/.codex/config.toml (no per-project MCP file), so copy this block +# into your global config — or use the `codex mcp add` one-liner in INTEGRATION.md. +# Replace the env values with your BrowserStack credentials. + +[mcp_servers.bstack] +command = "npx" +args = ["-y", "@browserstack/mcp-server@1.2.27-beta.1"] +env = { "BROWSERSTACK_USERNAME" = "your-username", "BROWSERSTACK_ACCESS_KEY" = "your-access-key", "O11Y_TFA_RCA_BASE_URL" = "" # optional: set only to target a staging tenant (default is production) } +startup_timeout_sec = 15 +tool_timeout_sec = 120 diff --git a/config/rca.config.json b/config/rca.config.json new file mode 100644 index 0000000..732c9e3 --- /dev/null +++ b/config/rca.config.json @@ -0,0 +1,61 @@ +{ + "$comment": "Central config for the /rca-build RCA harness. All formerly-hardcoded product/infra values live here. No kubectl/chitragupta/bifrost literals — connectors are discovered and probe-validated at the gate into the capability manifest (see skills/rca-build/references/evidence-routing.md). No reportFile: the plugin never writes a local RCA report — the full report lives on the Test Observability UI (triggerRcaReport).", + "mcpServerName": "bstack", + "concurrency": 5, + "turnCap": 6, + "turnMessageMaxChars": 5000, + "pollSoftPendingMs": 90000, + "reaperHeartbeatTtlSec": 600, + "errorSummaryMaxChars": 200, + "paths": { + "$comment": "State CSV path is ALWAYS derived per build via lib/csv-state.mjs csvPathFor(buildId, stateDir): the build id is in the filename (no cross-build collisions) and the default directory is OS temp (<tmpdir>/bstack-rca/), so the harness never pollutes the invoking workspace. Set stateDir only to retain the CSV somewhere specific (e.g. a CI artifact dir).", + "stateDir": "" + }, + "evidenceRouting": { + "test_logs": { + "owner": "tfa", + "skip": true + }, + "product_code": { + "capability": "github", + "discoveryHints": [ + "github-mcp", + "gh" + ] + }, + "deploy": { + "capability": "github", + "discoveryHints": [ + "github-mcp", + "gh" + ] + }, + "ci": { + "capability": "github", + "discoveryHints": [ + "github-mcp", + "gh" + ] + }, + "infra": { + "capability": "infra", + "discoveryHints": [] + }, + "k8s": { + "capability": "infra", + "discoveryHints": [] + }, + "kibana": { + "capability": "logs", + "discoveryHints": [] + }, + "metrics": { + "capability": "metrics", + "discoveryHints": [] + }, + "other": { + "capability": "other", + "discoveryHints": [] + } + } +} diff --git a/lib/coverage.mjs b/lib/coverage.mjs new file mode 100644 index 0000000..0266ceb --- /dev/null +++ b/lib/coverage.mjs @@ -0,0 +1,39 @@ +// Evidence-coverage stamp (ideation #6, v1 — the per-row coverage band; the +// build-level blast-radius digest is deferred). A RESOLVED RCA built with +// infra+logs+metrics all "unavailable" must not read like one with full +// evidence. The client (which routed every ask) stamps each row with a coverage +// vector and derives a coverage-capped confidence band the reviewer sees: +// "low confidence BECAUSE kibana was unavailable", not "low confidence, trust me". + +const BAND_ORDER = ["low", "medium", "high"]; + +// coverage classification from what was fulfilled vs. left unavailable. +export function classifyCoverage(asksFulfilled = [], asksUnavailable = []) { + const unavailable = [...new Set(asksUnavailable.filter(Boolean))]; + const fulfilled = [...new Set(asksFulfilled.filter(Boolean))]; + if (unavailable.length === 0) return "full"; + if (fulfilled.length > 0) return "partial"; + return "thin"; +} + +// Cap the band: full coverage keeps TFA's confidence; partial caps at medium; +// thin caps at low. Unknown/absent TFA confidence floors to low. +function capBand(tfaConfidence, coverage) { + const base = BAND_ORDER.includes(tfaConfidence) ? tfaConfidence : "low"; + const cap = coverage === "full" ? "high" : coverage === "partial" ? "medium" : "low"; + return BAND_ORDER[Math.min(BAND_ORDER.indexOf(base), BAND_ORDER.indexOf(cap))]; +} + +// The stamp written to a row at flip time. Returns { coverage, band, unavailable }. +export function coverageStamp({ + asksFulfilled = [], + asksUnavailable = [], + tfaConfidence = "unknown", +} = {}) { + const coverage = classifyCoverage(asksFulfilled, asksUnavailable); + return { + coverage, + band: capBand(tfaConfidence, coverage), + unavailable: [...new Set(asksUnavailable.filter(Boolean))], + }; +} diff --git a/lib/csv-state.mjs b/lib/csv-state.mjs new file mode 100644 index 0000000..152e51b --- /dev/null +++ b/lib/csv-state.mjs @@ -0,0 +1,266 @@ +// CSV write-ahead-log spine for the batch (D4 + ideation #7). The CSV is the +// single durable, resumable source of truth for "RCA over ALL failed tests": +// every test is a row, seeded `pending`, claimed by a worker, heartbeated while +// in flight, and flipped to a terminal state with its RCA. A reaper reclaims +// rows stranded by a crashed worker. +// +// Timestamps are passed in as `nowMs` (never read from the clock here) so this +// module is deterministic in tests AND usable from the auto-mode dynamic +// workflow, whose sandbox forbids Date.now(). +// +// In-session / in-workspace only — cross-session durability is deferred. Writes +// are synchronous read-modify-write; Node's single thread serializes them, which +// is sufficient for the in-process 5-concurrent workflow (true multi-process +// locking is out of scope). + +import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { tmpdir } from "node:os"; + +/** + * Canonical state-file path for one build's run. Two invariants (D-temp): + * 1. The BUILD ID IS IN THE FILENAME — runs over different builds can never + * collide/"resume" into each other's state. + * 2. Default location is OS TEMP (`<tmpdir>/bstack-rca/`), not the invoking + * workspace — a background harness must not pollute the repo it runs from. + * `stateDir` (config `paths.stateDir`) overrides the directory only — e.g. a CI + * job that wants the CSV as a retained artifact. Resume-safety is per build: + * same buildId → same path. + */ +export function csvPathFor(buildId, stateDir = "") { + const safe = String(buildId ?? "").replace(/[^A-Za-z0-9._-]/g, "_") || "unknown-build"; + const dir = stateDir && String(stateDir).trim() !== "" ? String(stateDir) : join(tmpdir(), "bstack-rca"); + return join(dir, `rca-state.${safe}.csv`); +} + +export const COLUMNS = [ + "buildId", + "testRunId", + "testName", + "failure_category", + "error_summary", + "file_path", + "cluster_id", + "rca_done", + "in_flight_worker", + "heartbeat_ts", + "threadId", + "turnId", + "last_evidence_digest", + "root_cause", + "failure_type", + "possible_fix", + "related_prs", + "coverage", + "confidence", + "timestamp", +]; + +export const PENDING = "pending"; +export const RESUMABLE = "pending-resume"; +// Truly done — never re-claimed, listed, or reaped. +const TERMINAL_STATES = new Set(["resolved", "blocked", "failed"]); +// Valid outcomes flip() may write. `pending-resume` is a *soft* terminal: this +// attempt ended (claim cleared) but the row stays resumable — it keeps its +// threadId/turnId and is picked back up by the next fan-out / resume pass. +const FLIP_STATES = new Set(["resolved", "blocked", "failed", RESUMABLE]); + +// ---- minimal RFC4180-ish CSV codec ---------------------------------------- + +function encodeField(value) { + const s = value == null ? "" : String(value); + if (/[",\r\n]/.test(s)) { + return `"${s.replace(/"/g, '""')}"`; + } + return s; +} + +function encodeRows(rows) { + const lines = [COLUMNS.join(",")]; + for (const row of rows) { + lines.push(COLUMNS.map((c) => encodeField(row[c])).join(",")); + } + return lines.join("\n") + "\n"; +} + +function parseCsv(text) { + const rows = []; + let field = ""; + let record = []; + let inQuotes = false; + for (let i = 0; i < text.length; i++) { + const ch = text[i]; + if (inQuotes) { + if (ch === '"') { + if (text[i + 1] === '"') { + field += '"'; + i++; + } else { + inQuotes = false; + } + } else { + field += ch; + } + } else if (ch === '"') { + inQuotes = true; + } else if (ch === ",") { + record.push(field); + field = ""; + } else if (ch === "\n" || ch === "\r") { + if (ch === "\r" && text[i + 1] === "\n") i++; + record.push(field); + rows.push(record); + field = ""; + record = []; + } else { + field += ch; + } + } + if (field.length > 0 || record.length > 0) { + record.push(field); + rows.push(record); + } + return rows; +} + +// ---- read / write ---------------------------------------------------------- + +export function readRows(csvPath) { + if (!existsSync(csvPath)) return []; + const text = readFileSync(csvPath, "utf8"); + const raw = parseCsv(text).filter((r) => r.some((c) => c.length > 0)); + if (raw.length === 0) return []; + const header = raw[0]; + return raw.slice(1).map((cells) => { + const row = {}; + header.forEach((col, idx) => { + row[col] = cells[idx] ?? ""; + }); + return row; + }); +} + +export function writeRows(csvPath, rows) { + const dir = dirname(csvPath); + if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true }); + writeFileSync(csvPath, encodeRows(rows), "utf8"); +} + +function emptyRow() { + return Object.fromEntries(COLUMNS.map((c) => [c, ""])); +} + +// ---- operations ------------------------------------------------------------- + +// Seed the CSV from a listTestIds(failed, includeFailureDetail) payload. Every +// row starts `pending`. Idempotent: existing rows are preserved (terminal rows +// are never reset; signature columns are refreshed on still-pending rows). New +// tests are appended. Returns the full row set. +export function seed(csvPath, buildId, tests) { + const existing = readRows(csvPath); + const byId = new Map(existing.map((r) => [String(r.testRunId), r])); + + for (const t of tests) { + const id = String(t.test_id ?? t.testRunId); + const sig = t.failure ?? {}; + const prior = byId.get(id); + if (prior) { + // Keep terminal results; only refresh signature on still-pending rows. + if (prior.rca_done === PENDING) { + prior.failure_category = sig.category ?? prior.failure_category; + prior.error_summary = sig.error_summary ?? prior.error_summary; + prior.file_path = sig.file_path ?? prior.file_path; + } + continue; + } + const row = emptyRow(); + row.buildId = buildId; + row.testRunId = id; + row.testName = t.test_name ?? t.testName ?? `Test ${id}`; + row.failure_category = sig.category ?? ""; + row.error_summary = sig.error_summary ?? ""; + row.file_path = sig.file_path ?? ""; + row.rca_done = PENDING; + byId.set(id, row); + existing.push(row); + } + + writeRows(csvPath, existing); + return existing; +} + +// Claim a pending row for `worker`. Refuses (returns false) if another worker +// already owns it. Returns true on success. +export function claim(csvPath, testRunId, worker, nowMs) { + const rows = readRows(csvPath); + const row = rows.find((r) => String(r.testRunId) === String(testRunId)); + if (!row) return false; + if (row.in_flight_worker && row.in_flight_worker !== worker) return false; + if (TERMINAL_STATES.has(row.rca_done)) return false; + row.in_flight_worker = worker; + row.heartbeat_ts = String(nowMs); + writeRows(csvPath, rows); + return true; +} + +export function heartbeat(csvPath, testRunId, worker, nowMs) { + const rows = readRows(csvPath); + const row = rows.find((r) => String(r.testRunId) === String(testRunId)); + if (!row || row.in_flight_worker !== worker) return false; + row.heartbeat_ts = String(nowMs); + writeRows(csvPath, rows); + return true; +} + +// Flip a row to a terminal state, recording the RCA fields and clearing the +// in-flight claim. `fields` carries any of: rca_done, root_cause, failure_type, +// possible_fix, related_prs, threadId, turnId, coverage, confidence, +// last_evidence_digest, cluster_id. +export function flip(csvPath, testRunId, fields, nowMs) { + // Enforce the contract: a flip must name a valid outcome. A partial flip with + // a missing/non-terminal rca_done would otherwise clear the claim yet leave the + // row `pending` — re-exposing it for a duplicate RCA that clobbers this result. + // Reject without mutating so the worker keeps its claim and the bug surfaces. + if (!FLIP_STATES.has(fields?.rca_done)) return false; + const rows = readRows(csvPath); + const row = rows.find((r) => String(r.testRunId) === String(testRunId)); + if (!row) return false; + for (const [k, v] of Object.entries(fields)) { + if (COLUMNS.includes(k)) { + row[k] = Array.isArray(v) ? v.join("; ") : (v ?? ""); + } + } + row.in_flight_worker = ""; + row.timestamp = String(nowMs); + writeRows(csvPath, rows); + return true; +} + +// Reclaim rows stranded in flight (heartbeat older than ttlSec) back to pending. +// Returns the testRunIds reclaimed. Run on startup before resuming a batch. +export function reaper(csvPath, ttlSec, nowMs) { + const rows = readRows(csvPath); + const reclaimed = []; + for (const row of rows) { + if (!row.in_flight_worker) continue; + if (TERMINAL_STATES.has(row.rca_done)) continue; + const hb = Number(row.heartbeat_ts); + const stale = !row.heartbeat_ts || nowMs - hb > ttlSec * 1000; + if (stale) { + row.in_flight_worker = ""; + row.rca_done = PENDING; + reclaimed.push(String(row.testRunId)); + } + } + if (reclaimed.length > 0) writeRows(csvPath, rows); + return reclaimed; +} + +// Rows still needing work: fresh/reclaimed `pending` AND `pending-resume` rows +// (soft-PENDING attempts that retain a threadId/turnId to resume). The fan-out +// work-list. Truly terminal rows (resolved/blocked/failed) are excluded. +export function pendingRows(csvPath) { + return readRows(csvPath).filter( + (r) => r.rca_done === PENDING || r.rca_done === RESUMABLE, + ); +} diff --git a/lib/evidence-cache.mjs b/lib/evidence-cache.mjs new file mode 100644 index 0000000..b9c9523 --- /dev/null +++ b/lib/evidence-cache.mjs @@ -0,0 +1,47 @@ +// Build-level evidence cache (ideation #2). "Diff since last green", "deploy +// timeline", "PRs in the suspect window" are properties of the BUILD, not the +// test — yet a naive loop re-fetches them per test. Compute once, cache by +// (repo, commit-range, evidenceType), and pre-seed every coordinator with the +// same grounded suspect window. Collapses N×M redundant git/infra calls to ~M. +// +// The cache is created fresh per run (function-scoped Map — never a module-level +// global), so it holds no cross-run/cross-user state: in-workspace, single +// session, multi-tenant-safe by construction. + +export function makeEvidenceCache() { + const store = new Map(); + const keyOf = (repo, range, evidenceType) => + `${repo ?? ""}@@${range ?? ""}@@${evidenceType ?? ""}`; + + return { + has(repo, range, evidenceType) { + return store.has(keyOf(repo, range, evidenceType)); + }, + get(repo, range, evidenceType) { + return store.get(keyOf(repo, range, evidenceType)); + }, + set(repo, range, evidenceType, value) { + store.set(keyOf(repo, range, evidenceType), value); + return value; + }, + // Compute-once: run `fn` only on a cache miss; reuse on every later call. + async compute(repo, range, evidenceType, fn) { + const k = keyOf(repo, range, evidenceType); + if (store.has(k)) return store.get(k); + const value = await fn(); + store.set(k, value); + return value; + }, + size() { + return store.size; + }, + }; +} + +// Resolve the baseline ref for the last-green→this-build delta. When there is no +// "last green" (e.g. a never-green flaky suite) fall back to a configured ref and +// flag it so the report can note the weaker grounding. +export function resolveBaseline(lastGreenRef, fallbackRef) { + if (lastGreenRef) return { ref: lastGreenRef, isFallback: false }; + return { ref: fallbackRef ?? null, isFallback: true }; +} diff --git a/lib/glimpse.mjs b/lib/glimpse.mjs new file mode 100644 index 0000000..38d7fb0 --- /dev/null +++ b/lib/glimpse.mjs @@ -0,0 +1,43 @@ +// Terse end-of-run summary — the ONLY in-client output of a /rca-build run. +// Deliberately a COMPLETION NOTICE, not a report: status counts + the UI link, +// nothing else. Root causes, culprit PRs, per-test analysis, cluster breakdowns +// live ONLY on the Test Observability dashboard (triggerRcaReport → viewReport). +// Do not reintroduce per-test cause/PR lines here — that is the whole point of +// the trim. Degrade, don't crash: missing fields are treated as absent. + +import { readRows } from "./csv-state.mjs"; + +// Map raw CSV rca_done states → the three buckets a human cares about. +function bucket(state) { + const s = (state || "").toLowerCase(); + if (s === "resolved") return "resolved"; + if (s === "pending" || s === "pending-resume") return "pending"; + return "failed"; +} + +// Render the completion summary. Returns a plain-text block: +// RCA analysis complete — build <id> +// <N> tests · <R> resolved · <P> pending · <F> failed +// (the caller appends the "Full report on the Test Observability UI: <link>" +// line from triggerRcaReport's viewReport — see SKILL.md Step 6). +export function renderGlimpse(rows, { buildId } = {}) { + const head = `RCA analysis complete${buildId ? ` — build ${buildId}` : ""}`; + if (!rows || rows.length === 0) { + return `${head}\nNo failed tests analyzed.\n`; + } + const counts = rows.reduce( + (acc, r) => { + acc[bucket(r.rca_done)] += 1; + return acc; + }, + { resolved: 0, pending: 0, failed: 0 }, + ); + const parts = [`${rows.length} test(s)`, `${counts.resolved} resolved`]; + if (counts.pending) parts.push(`${counts.pending} pending`); + if (counts.failed) parts.push(`${counts.failed} failed`); + return `${head}\n${parts.join(" · ")}\n`; +} + +export function renderGlimpseFromCsv(csvPath, opts = {}) { + return renderGlimpse(readRows(csvPath), opts); +} diff --git a/lib/loop.mjs b/lib/loop.mjs new file mode 100644 index 0000000..53e83da --- /dev/null +++ b/lib/loop.mjs @@ -0,0 +1,127 @@ +// Executable mirror of the ai-tfa-coordinator loop (agents/ai-tfa-coordinator.md). +// It drives the collaborative loop against an injected `submit` (real = the +// tfaRcaTurn MCP tool; tests = a recorded-turn replayer), so the loop mechanics — +// status branching, ask routing, gap degradation, turn-cap, one-thread, +// soft-PENDING — are tested rather than assumed. +// +// Double duty: this is ALSO the **sequential thin-client harness** — the third +// caller of the same contract, for MCP clients without workflows/subagents. +// Pure + dependency-light (imports only the routing registry). +// +// The loop is fully autonomous: an evidence gap ALWAYS degrades to an +// `unavailable` block back to TFA. There is no user-prompt path — the /rca-build +// gate closed before this loop ever runs. +// +// tfaRcaTurn returns TRIMMED terminal shapes: +// RESOLVED → { status, confidence, threadId, +// glimpse: { root_cause (≤220 chars), failure_type, related_prs }, +// viewRca } +// PENDING → { status, turnId, threadId } (soft-pending, resumable) +// NEEDS_INFO → { status, questions/asks/suggestions } (verbatim — the loop needs them) + +import { routeAsks } from "./routing.mjs"; + +function unavailableBlock(gap) { + const what = gap?.ask?.what ?? ""; + return [ + `ASK: ${what}`, + `TYPE: ${gap.evidenceType}`, + `FOUND: no`, + `SUMMARY: unavailable — no ${gap.capability} connector for this client.`, + ].join("\n"); +} + +// runRcaLoop drives one test to a terminal RCA_OUTPUT object. +// +// submit({ testRunId, message, threadId, turnId }) → Promise<turn> (tfaRcaTurn shape) +// gather(routedGatherEntry) → Promise<string> (one digest block) +export async function runRcaLoop({ + testRunId, + firstMessage = "", + submit, + config = {}, + manifest = {}, + gather = async () => "", + turnCap = config?.turnCap ?? 6, +}) { + if (testRunId == null || Number.isNaN(Number(testRunId))) { + return { + testRunId: String(testRunId), + status: "failed", + root_cause: "no testRunId provided", + turns_used: 0, + asks_fulfilled: [], + asks_skipped: [], + asks_unavailable: [], + }; + } + + let threadId; + let turnId; + let turns = 0; + let message = firstMessage; + const fulfilled = new Set(); + const skipped = new Set(); + const unavailable = new Set(); + + const out = (status, turn, note) => { + const glimpse = turn?.glimpse ?? {}; + return { + testRunId: String(testRunId), + status, + confidence: turn?.confidence ?? "unknown", + root_cause: status === "RESOLVED" ? (glimpse.root_cause ?? "") : (note ?? ""), + failure_type: glimpse.failure_type ?? "", + related_prs: glimpse.related_prs ?? [], + view_rca: turn?.viewRca ?? "", + threadId: threadId ?? null, + turnId: turnId ?? null, + turns_used: turns, + asks_fulfilled: [...fulfilled], + asks_skipped: [...skipped], + asks_unavailable: [...unavailable], + }; + }; + + while (true) { + turns++; + const turn = await submit({ testRunId, message, threadId, turnId }); + threadId = turn.threadId ?? threadId; + + if (turn.status === "RESOLVED") return out("RESOLVED", turn); + if (turn.status === "PENDING") { + turnId = turn.turnId ?? turnId; + return out("PENDING", turn, "soft-pending"); + } + + // NEEDS_INFO. Check the turn-cap BEFORE gathering — evidence assembled on a + // turn we will never submit is wasted work (and a side-effecting gather() + // would run for nothing). + if (turns >= turnCap) return out("PENDING", turn, "turn-cap"); + + // Route + fulfill. Gaps degrade to `unavailable` — never a user prompt. + const buckets = routeAsks(turn.asks ?? [], config, manifest); + const blocks = []; + for (const s of buckets.skip) skipped.add(s.evidenceType); + for (const g of buckets.gather) { + blocks.push(await gather(g)); + fulfilled.add(g.evidenceType); + } + for (const gap of buckets.gap) { + unavailable.add(gap.evidenceType); + blocks.push(unavailableBlock(gap)); + } + + message = blocks.join("\n\n"); + } +} + +// Replay helper for tests: returns a submit() that yields recorded turns in order. +export function replaySubmit(turns) { + let i = 0; + return async () => { + const turn = turns[Math.min(i, turns.length - 1)]; + i++; + return turn; + }; +} diff --git a/lib/routing.mjs b/lib/routing.mjs new file mode 100644 index 0000000..ba3d699 --- /dev/null +++ b/lib/routing.mjs @@ -0,0 +1,107 @@ +// Evidence-routing registry (D3). Maps a TFA `ask.evidenceType` onto an +// action, given the run's validated capability manifest. Pure + dependency-free +// so it is testable and reusable by the batch workflow, subagents, and the +// sequential harness alike. +// +// `test_logs` is the TFA agent's own evidence and is always skipped. Every +// other type routes to a capability; whether that capability is *available* is +// decided by the manifest (built once per run — see U6 / buildManifest). + +import { readFileSync } from "node:fs"; + +export const TEST_LOGS = "test_logs"; + +const PRIORITY_RANK = { high: 0, medium: 1, low: 2 }; + +// Load and parse config/rca.config.json from an absolute or cwd-relative path. +export function loadConfig(configPath) { + return JSON.parse(readFileSync(configPath, "utf8")); +} + +// Order a turn's asks high → medium → low (unknown priority sorts last). +export function orderAsks(asks = []) { + return [...asks].sort( + (a, b) => + (PRIORITY_RANK[a?.priority] ?? 99) - (PRIORITY_RANK[b?.priority] ?? 99), + ); +} + +// Classify one ask. Returns one of: +// { action: "skip", ... } — test_logs / TFA-owned; the coordinator emits nothing +// { action: "gather", ... } — a capability is available; gather + digest +// { action: "gap", ... } — no valid connector; the caller emits an +// "unavailable" block back to TFA (never a prompt) +// +// `manifest` shape: { [capability]: { available: boolean, via?: string } }. +export function routeAsk(ask, config, manifest = {}) { + const evidenceType = ask?.evidenceType ?? "other"; + const routing = config?.evidenceRouting ?? {}; + const entry = routing[evidenceType] ?? routing.other ?? { capability: "other" }; + + if (entry.skip || entry.owner === "tfa") { + return { evidenceType, action: "skip", reason: "tfa-owned" }; + } + + const capability = entry.capability ?? "other"; + const cap = manifest[capability]; + if (cap && cap.available) { + return { + evidenceType, + action: "gather", + capability, + via: cap.via ?? null, + }; + } + + return { + evidenceType, + action: "gap", + capability, + discoveryHints: entry.discoveryHints ?? [], + reason: "no-capability", + }; +} + +// Split a turn's asks into the three buckets, in priority order. The +// coordinator gathers `gather`, emits an "unavailable" block for each `gap`, +// and records `skip` (test_logs) without emitting anything. +export function routeAsks(asks, config, manifest = {}) { + const ordered = orderAsks(asks); + const buckets = { skip: [], gather: [], gap: [] }; + for (const ask of ordered) { + const routed = routeAsk(ask, config, manifest); + buckets[routed.action].push({ ask, ...routed }); + } + return buckets; +} + +// ---- capability manifest (ideation #3) ------------------------------------- + +// Build the capability manifest ONCE per run from the capabilities the client +// agent actually discovered. `discovered` is a list of +// { capability, via } the orchestrator collected by asking "what skills/tools +// are available?". Every capability the routing registry references (except the +// TFA-owned test_logs) appears in the manifest, marked available iff discovered. +// Declaring this to TFA lets it avoid asking for evidence the client can't get. +export function buildManifest(config, discovered = []) { + const byCap = new Map(discovered.map((d) => [d.capability, d])); + const manifest = {}; + for (const entry of Object.values(config?.evidenceRouting ?? {})) { + if (entry.skip || entry.owner === "tfa") continue; + const cap = entry.capability; + if (!cap || cap in manifest) continue; + const found = byCap.get(cap); + manifest[cap] = found + ? { available: true, via: found.via ?? null } + : { available: false, via: null }; + } + return manifest; +} + +// Capabilities that will be unavailable this run — declared to the user up front +// ("infra + metrics not available") and to TFA so it plans asks around them. +export function unavailableCapabilities(manifest) { + return Object.entries(manifest) + .filter(([, v]) => !v.available) + .map(([cap]) => cap); +} diff --git a/lib/signature.mjs b/lib/signature.mjs new file mode 100644 index 0000000..42dc0ae --- /dev/null +++ b/lib/signature.mjs @@ -0,0 +1,78 @@ +// Failure-signature clustering (ideation #1). A red build's N failures usually +// trace to a handful of causes; clustering collapses the expensive evidence hunt +// to O(distinct causes). The signature is computed from the trimmed failure +// detail U1 surfaces on each listTestIds row (category + first error line + file +// path) — no extra probe turns. +// +// Dependency-free + deterministic (no crypto, no clock, no random) so it is +// usable from the auto-mode workflow sandbox and trivially testable. + +// Normalize a string for signature comparison: lowercase and fold the volatile +// tokens that make two instances of the SAME failure look different (ids, +// timestamps, hex/uuids, line:col, bare numbers). +export function normalize(value) { + return String(value ?? "") + .toLowerCase() + .replace(/\b\d{4}-\d{2}-\d{2}[t ]\d{2}:\d{2}:\d{2}\S*/g, "<ts>") // ISO timestamps + .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g, "<uuid>") + .replace(/0x[0-9a-f]+/g, "<hex>") // memory addresses + .replace(/:\d+(:\d+)?\b/g, ":<line>") // file:line(:col) + .replace(/\d+/g, "<n>") // remaining numbers (incl. unit-suffixed, e.g. 3000ms) + .replace(/\s+/g, " ") + .trim(); +} + +// The signature triple: normalized category | error summary | file path. +export function computeSignature(row) { + const category = normalize(row.failure_category); + const error = normalize(row.error_summary); + const file = normalize(row.file_path); + const sig = `${category}|${error}|${file}`; + return sig.replace(/\|/g, "").trim().length === 0 ? "" : sig; +} + +// Deterministic short id for a signature string (FNV-1a → base36). +function hashId(s) { + let h = 0x811c9dc5; + for (let i = 0; i < s.length; i++) { + h ^= s.charCodeAt(i); + h = Math.imul(h, 0x01000193); + } + return (h >>> 0).toString(36); +} + +// A stable representative for a cluster: prefer a non-flaky member (a flaky test +// is a poor exemplar), then the smallest testRunId. Deterministic. +export function selectRepresentative(members) { + return [...members].sort((a, b) => { + const aFlaky = a.is_flaky === "true" || a.is_flaky === true ? 1 : 0; + const bFlaky = b.is_flaky === "true" || b.is_flaky === true ? 1 : 0; + if (aFlaky !== bFlaky) return aFlaky - bFlaky; + return Number(a.testRunId) - Number(b.testRunId); + })[0]; +} + +// Cluster rows by signature. Mutates each row's `cluster_id`. Rows with no +// signal (empty signature) become their own singleton (never merged into a +// catch-all). Returns { rows, clusters } where each cluster carries its +// representative + siblings. +export function clusterRows(rows) { + const groups = new Map(); + + for (const row of rows) { + const sig = computeSignature(row); + const id = sig === "" ? `solo-${row.testRunId}` : `c-${hashId(sig)}`; + row.cluster_id = id; + if (!groups.has(id)) groups.set(id, { cluster_id: id, signature: sig, members: [] }); + groups.get(id).members.push(row); + } + + const clusters = []; + for (const group of groups.values()) { + const representative = selectRepresentative(group.members); + const siblings = group.members.filter((m) => m !== representative); + clusters.push({ ...group, representative, siblings }); + } + + return { rows, clusters }; +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..27344a7 --- /dev/null +++ b/package.json @@ -0,0 +1,10 @@ +{ + "name": "tfa-rca-plugin", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "Generic multi-client RCA agent plugin harness", + "scripts": { + "test": "node --test" + } +} diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md new file mode 100644 index 0000000..140b2a9 --- /dev/null +++ b/skills/rca-build/SKILL.md @@ -0,0 +1,238 @@ +--- +name: rca-build +description: Single-gate autonomous batch RCA over every failed test of a BrowserStack build via tfaRcaTurn. One gate (connector validation + assumed intake), then fully autonomous — clusters failures, routes evidence, triggers the dashboard report. Args: build id, optional PR URLs / repo hints. +--- + +# rca-build — single-gate autonomous RCA over a build + +Drives the `tfaRcaTurn` collaborative loop over **every failed test** of a build +and lands a per-test RCA in the TRA (Test Observability) dashboard. **TFA owns +logs; the client agent owns everything else** (product code, infra/runtime, logs, +metrics, deploy, ci) — routed by capability, generic over product and infra. + +This skill is the **build-level orchestrator** (`ai-tfa-orchestrator` role). It +never calls `tfaRcaTurn` itself — it dispatches the `ai-tfa-coordinator` +(test-level) per test/cluster member, which drives the loop and lets TFA author +the dashboard RCA. **The full RCA report lives on the Test Observability UI, not +in Claude** — this run's job is to feed it, then surface a terse glimpse and the +link. + +There is exactly **one mode**: autonomous. There is exactly **one gate** (Step +1) before execution. After the gate closes, **the run never asks the user +anything again.** + +Config (concurrency, turn-cap, paths, evidence registry) lives in +`config/rca.config.json`. State lives in the CSV/WAL spine (`lib/csv-state.mjs`). + +## Step 0 — input + +Parse the build id from the invocation args. Accepted forms: a bare build id, a +`build_id=<id>` token, or a build dashboard link (extract the id). Also accept +any **PR URLs** and **repo hints** (product/automation repo names or paths) the +user supplies — carry them into Gate Part B as pre-answered intake. + +- No build id present: + - interactive session → fold "which build?" into the gate's single + consolidated question (Step 1, Part B). It is the only genuinely + load-bearing field. + - **headless (`claude -p`) → end immediately (fail fast).** + +## Step 1 — THE GATE (one gate, two parts, closes once) + +Everything the run could possibly need from the user is settled here, in one +pass. The gate has two parts; both run before any RCA work starts. + +### Part A — connector discovery + validation + +Enumerate every connector relevant to test RCA: + +- from `config/rca.config.json` → `evidenceRouting`: **github** + (product_code/deploy/ci), **infra** (whatever runtime the user has — k8s, + ECS, docker, Nomad, plain VMs, PM2, …), **logs** (kibana or any log store), + **metrics**, **other**; +- plus any connector-shaped skills / MCP servers present in the session + (a log-search MCP, a metrics MCP, an infra skill, …). + +**Validate** each with a cheap probe — discovery alone is not enough: + +| Connector | Probe | +|---|---| +| github | `gh auth status` (or a GitHub MCP tool listed) | +| infra | ANY runtime connector the user has — probe what exists, never assume one: `kubectl version --request-timeout=5s`, `docker ps`, `aws ecs list-clusters`, `nomad status`, `pm2 ls`, or an infra-shaped skill/MCP tool. Record the KIND in the manifest (`via: kubectl \| docker \| ecs \| …`) | +| logs | a log-search skill/MCP tool actually listed in the session | +| metrics | a metrics skill/MCP tool actually listed in the session | +| other | best-effort; default `absent` | + +Output the **validated capability manifest**: `connector → valid | invalid | +absent` (`lib/routing.mjs` → `buildManifest`; `valid` maps to +`available: true`). An `invalid` or `absent` connector is a **recorded gap** — +declared to the user in the gate summary and to TFA on the first turn ("I don't +have logs/metrics access") — **never a blocker**. The run always proceeds. + +### Part B — requirements (assume first, ask once, at most) + +Intake fields: product repo, automation (test) repo, working branch, default +branch, the PRs in play, and the build id. **Resolve every field by ASSUMPTION +wherever possible** — this is an assumption-OK workflow; less user interaction +is the point: + +- invocation args (build id, PR URLs, repo hints from Step 0), +- `gh repo view` / git remotes for the repos, +- the current branch for the working branch, +- cheap inference (e.g. the automation repo is the cwd if it holds the tests). + +**Product-repo corroboration (do NOT skip).** The product repo must plausibly +be the *system under test for THIS build's failures* — not merely a repo name +found lying around. A repo mentioned only in workspace docs/READMEs is a **weak +hint, never an assumption**: cross-check it against the failure signatures +(discovery runs first if needed) — do the failing area, files, or error strings +relate to that repo's domain? If they don't (e.g. the failures are self-healing +`healedElement is null` cases but the only named repo is an observability API), +the doc-sourced repo is discarded — never carry it (or its PRs) into the +manifest as a settled product repo. + +When corroboration leaves **no** product repo, decide by whether a human can help: +- **PRs were supplied** → treat those as the suspect surface; product repo is + derived from them. No question needed. +- **No PRs, interactive session** → the product repo is now **non-assumable AND + load-bearing** (without it the mandatory culprit-PR hunt is dead), so it earns + the single consolidated gate question below — ask it; don't silently degrade. +- **No PRs, headless** → record the gap ("product repo: unknown") and proceed + RCA-only; every culprit-PR hunt reports "no culprit PR identified". + +Record each assumption in the gate summary (format: +`templates/gate-summary.md`; worked example: `examples/sample-run.md`) +("assumed product repo = +`org/obs-api` from git remote"). A field that cannot be assumed is recorded as +"none" and the run proceeds RCA-only for it — **unless** it is both genuinely +non-assumable AND load-bearing. In practice that set is: the build id; **the +product repo when it could not be corroborated and no PRs were supplied** (see +above — without it the culprit-PR hunt cannot run); and rarely an ambiguous repo +when PRs were supplied. Those, and only those, may be asked **ONCE, in a single +consolidated question at gate close** — e.g. *"Failures look like `<domain>`; +which repo owns that code? (reply 'none' → I'll RCA without culprit-PR +attribution)."* Never a second question. **Headless: skip asking entirely; +record the gaps.** + +### Gate close + +Print a one-screen summary: resolved intake (with assumptions marked) + the +validated capability manifest (with gaps named). Then the gate closes. + +**AFTER THE GATE CLOSES, THE RUN NEVER ASKS THE USER ANYTHING AGAIN.** RCA +execution is fully autonomous: every downstream evidence gap becomes an +`unavailable` block back to TFA (best-effort finalize), never a prompt. + +## Step 2 — discovery + +Call the bundled MCP tool: + +``` +listTestIds(buildId=<id>, status="failed", includeFailureDetail=true) +``` + +`includeFailureDetail=true` returns each row's trimmed failure signature +(`failure.{category, error_summary, file_path, …}`) — the seed for clustering, +so no per-test probe turns are needed. + +Resolve the state file with `lib/csv-state.mjs` → `csvPathFor(buildId, +config.paths.stateDir)` — the **build id is in the filename** and the default +directory is **OS temp** (`<tmpdir>/bstack-rca/rca-state.<buildId>.csv`), so +different builds can never collide and the invoking workspace stays clean. Pass +this exact path to the fan-out workflow as `csvPath`. + +Seed the CSV/WAL spine from the payload (`lib/csv-state.mjs` → `seed`): one row +per failed test, every row `rca_done=pending`, signature columns populated. +Re-running `seed` on an existing CSV is idempotent and preserves terminal rows +(resume-safe — same build id → same path). If `listTestIds` returns empty → +write an empty CSV, report "no failed tests", stop. + +## Step 3 — failure-signature clustering (see references/clustering.md) + +Compute a failure signature per row and assign `cluster_id` (`lib/signature.mjs`). +Each cluster gets one **representative** (full multi-turn loop) and `N−1` +**siblings** (pre-seeded one-turn confirm against their own logs). This collapses +the expensive evidence hunt to O(distinct causes) while every test still lands a +per-test RCA. Singleton clusters are just plain per-test loops. + +## Step 4 — build-evidence pre-compute (see references/evidence-routing.md) + +Once, before fan-out (the capability manifest already exists from Gate Part A — +reuse it, do not re-discover): + +- **Build-level evidence** — compute the last-green→this-build delta (diff, + deploy timeline, suspect-PR window) **once** and pre-seed every coordinator + with the same grounded window. Cache by `(repo, commit-range)`. No "last green" + baseline (never-green suite) → fall back to a configured baseline ref and log it. + +## Step 5 — fan-out (fully autonomous) + +Drive the cluster work-list, **`concurrency` (default 5) at a time**: +representatives deep, siblings one-turn-confirm. Eagerly persist to the CSV/WAL +(claim → heartbeat → flip) so the run is resumable. + +- Claude Code → run the dynamic workflow `workflows/rca-batch.mjs` + (script-orchestrated; gap → "unavailable" back to TFA → best-effort finalize). +- Hosts without the Workflow runtime → dispatch `tfa-rca:ai-tfa-coordinator` + subagents ≤ `concurrency` at a time, or drive the sequential harness + `lib/loop.mjs` (`runRcaLoop`). Same contract, same no-prompt rule. + +Subagents/coordinators return compact `RCA_OUTPUT` blocks, never transcripts. A +coordinator that dies becomes a recorded `failed` row — one stuck test never +sinks the batch (partial-first). No path ever prompts the user (the gate is +closed). + +**Application bugs need a culprit PR.** Whenever a test's RCA classifies as +PRODUCT_BUG / application bug, the coordinator MUST hunt the culprit PR via the +github connector (deploy timeline vs last-pass window, changed paths vs failure +signature — `references/github-evidence.md`) and feed the PR link(s) to TFA in +the turn message so the dashboard RCA's `related_prs` populates. An +application-bug RCA with no GitHub PR link is **incomplete**: keep digging until +the turn cap; if still none, the turn must explicitly state "no culprit PR +identified after <what was searched>" and the CSV row records the gap. + +## Step 6 — finish: glimpse + dashboard report (NO local report) + +This plugin **never renders or writes a local RCA report, and never surfaces RCA +detail in Claude.** The in-Claude output is a two-line completion notice plus the +link — that is all. When every row is terminal: + +1. Print the **completion summary** from the CSV (`lib/glimpse.mjs` → + `renderGlimpse`): `RCA analysis complete — build <id>` + a status count line + (`<N> tests · <R> resolved · <P> pending · <F> failed`). **Nothing per-test.** +2. Call **`triggerRcaReport(buildUuid=<build id>)`** (add `force=true` only to + re-run over an existing completed report). +3. Print the link line, verbatim shape: + + ``` + Full report on the Test Observability UI: <viewReport> + ``` + +**Do NOT print** root causes, culprit/related PRs, cluster breakdowns, per-test +analysis, confidence rationales, or a per-test table — root_cause, related_prs, +suspect_signals and the like are for the CSV + the dashboard ONLY. If a human +wants the "why", they open the link. Claude's job here is "analysis complete → +report is at <link>", not to re-narrate the RCA the BrowserStack agent authored. + +## Resume + +On startup, run the reaper (`lib/csv-state.mjs` → `reaper`) to reclaim rows +stranded `in_flight` by a crashed worker (heartbeat older than +`reaperHeartbeatTtlSec`) back to `pending`, then re-point fan-out at the CSV. +Live `threadId`/`turnId` resume the prior thread; dead threads re-run from +pending. Resuming a run does **not** reopen the gate — no new questions. +(In-session only — cross-session durability is deferred.) + +## Hard rules + +- Exactly one gate. At most one consolidated question, at gate close. **After + the gate closes, never ask the user anything.** +- An invalid/absent connector is a recorded gap, never a blocker. +- Headless + missing build id → end immediately. Headless never asks. +- Never call `tfaRcaTurn` from this skill — always via the `ai-tfa-coordinator`. +- Every failed test must end terminal in the CSV — partial-first, no abort-on-one-failure. +- Never gather `test_logs` — TFA owns logs. +- Never render/write a local RCA report — glimpse table + `triggerRcaReport` + + the Test Observability UI link only. +- A PRODUCT_BUG RCA without a GitHub PR link is incomplete — dig until the turn + cap, else state what was searched and record the gap. diff --git a/skills/rca-build/examples/sample-run.md b/skills/rca-build/examples/sample-run.md new file mode 100644 index 0000000..5efa6e2 --- /dev/null +++ b/skills/rca-build/examples/sample-run.md @@ -0,0 +1,77 @@ +# Example — one full run (fictional data, matches the recorded-turn fixtures) + +Invocation: `/rca-build awswx…fw2` (build id given; nothing else passed). + +## 1. Gate closes (the only user-visible checkpoint) + +``` +GATE CLOSED — capability manifest: + github ✅ valid (gh, authed) · infra ✅ valid (via kubectl, ctx staging-euc1) · logs ❌ absent · metrics ❌ absent + +Intake: + build id: awswx…fw2 (given) + product repo: acme/obs-api (assumed — from git remote) + automation repo: acme/obs-e2e (assumed — cwd holds the tests) + working branch: main (assumed — current branch) + default branch: main (assumed — origin HEAD) + PRs in play: none (gap) + +Gaps declared to TFA: logs, metrics +Proceeding autonomously: discovery → clustering → fan-out (concurrency 5, turn-cap 6). +``` + +## 2. A NEEDS_INFO turn answered (what the coordinator sends back) + +TFA asked: *"Did request-validation on POST /builds change since last green?"* +(`evidenceType: product_code`, priority high). The coordinator replies on the +same `threadId`: + +``` +ASK: Did request-validation on POST /builds change since last green? +TYPE: product_code +FOUND: yes +SUMMARY: Yes — the buildName validator was tightened to reject empty strings in the +suspect window. One PR touches the failing path; falsification below. +LINK: https://github.com/acme/obs-api/pull/7421 + +SUSPECT: + pr: #7421 + files: src/validators/build.ts + hunks: `- allowEmpty: true` → `+ allowEmpty: false` (validator schema) + author: jdoe + merged_at: 2026-07-01T09:14Z vs last_green: 2026-07-01T02:10Z vs started_at: 2026-07-01T21:40Z + verdict: supported + link: https://github.com/acme/obs-api/pull/7421 + +SUSPECT: + pr: #7418 + files: src/routes/builds.ts + hunks: logging middleware reorder only + author: asmith + merged_at: 2026-06-30T18:02Z vs last_green: 2026-07-01T02:10Z vs started_at: 2026-07-01T21:40Z + verdict: ruled-out (shipped-after check passed but no-path-overlap — hunks never touch the validator) + link: https://github.com/acme/obs-api/pull/7418 + +ASK: Full run logs for test 39 +TYPE: test_logs +FOUND: no +SUMMARY: out-of-scope — TFA owns test logs; skipped by contract. +``` + +## 3. Terminal output (glimpse only — NO local report) + +``` +RCA analysis complete — build awswx…fw2 +7 test(s) · 6 resolved · 1 pending + +Full report on the Test Observability UI: +https://automation.browserstack.com/builds/awswx…fw2?tab=ai_report&subTab=aitfa +``` + +That is the **entire** in-Claude output. No root causes, no culprit PRs, no +per-test table — those are on the dashboard, authored by the BrowserStack agent. +The RESOLVED turn shown earlier is the coordinator↔TFA exchange (internal to the +loop), not something re-printed to the user at the end. + +State file: `<tmpdir>/bstack-rca/rca-state.awswx…fw2.csv` (resume-safe; re-run +the same build id to pick up the PENDING row). diff --git a/skills/rca-build/references/clustering.md b/skills/rca-build/references/clustering.md new file mode 100644 index 0000000..1ac5866 --- /dev/null +++ b/skills/rca-build/references/clustering.md @@ -0,0 +1,60 @@ +# Failure-signature clustering + +Why: a red build's N failures usually trace to a handful of causes (one bad +PR/deploy/shared helper). Running the full collaborative loop once per *cause* +instead of once per *test* turns the dominant cost from **O(tests) → O(distinct +causes)** — the only thing that makes "RCA for ALL failed tests, even thousands" +feasible. But **every failed test must still show a per-test RCA in the TRA +dashboard**, so clustering collapses the *evidence hunt*, not the *output*. + +The logic lives in `lib/signature.mjs`; this file is the protocol. + +## The signature + +Computed from the trimmed failure detail `listTestIds(includeFailureDetail=true)` +already returns on each row — **no extra probe turns**: + +``` +signature = normalize(failure_category) | normalize(error_summary) | normalize(file_path) +``` + +`normalize` folds the volatile tokens that make two instances of the *same* +failure look different: ISO timestamps, UUIDs, hex/memory addresses, `file:line:col`, +and bare numbers. So `timeout after 3000ms on node-7` and `timeout after 5000ms +on node-2` share a signature. + +A row with **no signal** (empty category, error, and path) is **not** merged into +a catch-all — it becomes its own singleton (`solo-<testRunId>`). Better an +un-clustered test than a wrong cluster. + +## Representative + siblings + +Each cluster gets: + +- **Representative** — a stable exemplar (non-flaky preferred, then smallest + `testRunId`). Runs the **full multi-turn `ai-tfa-coordinator` loop** → + confirmed root cause + culprit `related_prs`. +- **Siblings** (`N−1`) — each runs its **own** coordinator, **pre-seeded** with + the representative's `root_cause` + suspect PRs. TFA confirms the hypothesis + **against that sibling's own logs in a single turn** → a logs-grounded per-test + RCA in the dashboard at minimal cost. + +Net cost per cluster: **1 deep investigation + (N−1) one-turn confirms.** + +## The safeguard — never blindly inherit + +Distinct failures can share an error string. A sibling's pre-seed turn is a +*hypothesis to confirm*, not a verdict to copy: + +- TFA `RESOLVED`s the sibling in one turn → logs-grounded inheritance, cheap. +- TFA returns `NEEDS_INFO` (the hypothesis does not hold for this + test's logs) → the sibling **falls back to its own full loop**. The + representative's cause is never stamped onto a sibling without log confirmation. + +This keeps correctness independent of the cost optimization: worst case, every +sibling runs its own full loop (same as no clustering); best case, one deep run +covers the whole cluster. + +## Singletons + +A cluster of one is just a plain per-test loop — no pre-seed, no confirm step. diff --git a/skills/rca-build/references/evidence-routing.md b/skills/rca-build/references/evidence-routing.md new file mode 100644 index 0000000..05b4435 --- /dev/null +++ b/skills/rca-build/references/evidence-routing.md @@ -0,0 +1,160 @@ +# Evidence Routing + +Load this file **before fulfilling any `NEEDS_INFO` ask** in the per-test RCA +loop (`agents/ai-tfa-coordinator`). It maps each TFA `evidenceType` to a +**capability** (not a hardcoded tool), and defines the **digest** the coordinator +submits on the next turn. + +The core contract: **TFA owns logs; the client agent owns everything else.** The +coordinator never seeds logs and never fulfills a `test_logs` ask. Every other +`evidenceType` routes to a capability that is gathered via **whatever skill/tool +the client actually has** for it (discovered **and validated** once into the +capability manifest — see `SKILL.md` § Gate Part A). There are **no `kubectl` / +`chitragupta` / +`bifrost` literals here** — that is the whole point of going generic. + +The registry logic lives in `lib/routing.mjs` (`routeAsk` / `routeAsks`); this +file is the human/agent-facing contract for the digest and the size caps. + +--- + +## How a turn's asks are processed + +A `NEEDS_INFO` turn returns `asks: TfaAsk[]`, each `{ what, why, evidenceType, +priority }`. For each ask, in descending `priority` (`high` → `medium` → `low`): + +1. Route the `evidenceType` (via `lib/routing.mjs` → the config registry + + capability manifest). The result is one of three actions: + - **skip** — `test_logs` (TFA-owned). Gather nothing; record in `asks_skipped`. + - **gather** — a capability is available. Run its discovered skill/tool scoped + by `what` / `why`, then digest the result into one ask block. + - **gap** — no valid connector for that `evidenceType` (the gate recorded it + as `invalid`/`absent`). Emit an `unavailable` block back to TFA — **never + prompt the user** (the gate is closed; the run is autonomous). +2. Concatenate the per-ask blocks into the next-turn `message` and resubmit on + the same `threadId`. + +An ask that cannot be fulfilled is **never silently dropped** — it becomes a +`not-found` / `unreachable` / `unavailable` block so TFA can reason about the gap. + +--- + +## Routing table (capability, not tool) + +`evidenceType` literals are exactly those `tfaRcaTurn` emits: `test_logs`, +`product_code`, `infra` (TFA may still spell it `k8s` — both route the same), +`kibana`, `metrics`, `deploy`, `ci`, `other`. + +| `evidenceType` | Capability | Gathered via (discovered at runtime) | +|---|---|---| +| `test_logs` | — (TFA, skip) | never gathered; TFA self-serves from its own log access | +| `product_code` | `github` | the client's GitHub capability — **GitHub MCP if present, else `gh`** (see `references/github-evidence.md`) | +| `deploy` | `github` | deploy timeline via the GitHub capability (releases/tags + deploy record) | +| `ci` | `github` | CI config + run history via the GitHub capability | +| `infra` / `k8s` | `infra` | **whatever runtime connector the user has** — k8s/EKS, ECS, docker, Nomad, plain VMs, PM2, … Discovered and probed at the gate, NEVER assumed to be Kubernetes; the manifest records the kind (`via`) | +| `kibana` | `logs` | whatever log-search skill the client has (kibana or other) | +| `metrics` | `metrics` | whatever metrics skill the client has | +| `other` | `other` | best-effort by ask text; else a `not-found` block | + +The mapping is data in `config/rca.config.json` (`evidenceRouting`), so a +different deployment can remap `evidenceType → capability` without code changes. + +**Deployment-state guard:** a suspect PR only matters if its code was actually +live in the run's env at the failure window. If you can cheaply confirm it was +not deployed / behind an OFF flag, say so in the digest rather than feeding TFA a +suspect that could not have caused the failure. (Full protocol: U9 / +`references/github-evidence.md`.) + +--- + +## Digest format + +The single most important discipline: **digested input, not raw dumps.** Every +turn's `message` loads into the agent's context *and* is sent to TFA; a raw log +tail or full PR diff blows both budgets and degrades TFA's reasoning. Supply the +*findings*, not the *haystack*. + +### Per-ask block shape — `ask → found → snippet/link` + +**The canonical fillable format lives in +[`../templates/evidence-block.md`](../templates/evidence-block.md)** (fulfilled +and unfulfillable variants) — copy it, don't retype it. Shape: +`ASK / TYPE / FOUND: yes|no|partial / SUMMARY ≤400 / SNIPPET (caps below) / LINK`. + +- `SUMMARY` is the answer. `SNIPPET` is the *minimum* evidence backing it. `LINK` + lets TFA (or a human) verify without the bytes living in the message. +- Prefer **LINK over SNIPPET** whenever a permalink fully carries the evidence. + +### Size caps (hard ceilings — truncate, never exceed) + +| Field / scope | Soft target | Hard ceiling | On exceed | +|---|---|---|---| +| `SUMMARY` | ≤ 300 chars | 400 chars | Tighten to the finding; drop restatement of the ask | +| `SNIPPET` per ask | ≤ 20 lines | 40 lines | Keep the load-bearing lines; replace the rest with `… (N lines elided — see LINK)` | +| Code diff in a `product_code` snippet | ≤ 1 hunk | 3 hunks | Show changed lines + 3 lines context; link the full PR | +| Whole next-turn `message` | ≤ 200 lines | 400 lines (and ≤ `turnMessageMaxChars`) | Drop `low`-priority asks first; keep every `high` ask's block | +| Asks fulfilled per turn | all `high` + `medium` | — | Defer `low` asks to a later turn rather than truncating a `high` ask | + +Truncation rule of thumb: **never truncate a `high`-priority ask's block to fit a +`low`-priority one.** Drop the low block whole; keep the high block intact. The +whole-message ceiling also honors `turnMessageMaxChars` from +`config/rca.config.json` (the tool caps `message` at 5000 chars). + +### What never goes in a digest + +- Raw log tails, full log output, full file contents, full PR diffs — link or excerpt. +- `test_logs` content of any kind (TFA owns it). +- Credentials, tokens, internal hostnames, or any secret surfaced by an env/secret dump. +- Speculation dressed as a finding. If `FOUND: no`, say what was checked; do not invent a cause. + +--- + +## Unfulfillable asks — report, don't drop + +``` +ASK: <verbatim what> +TYPE: <evidenceType> +FOUND: no +SUMMARY: not-found | unreachable | unavailable | out-of-scope — <one line: what was checked or why blocked> +``` + +- `not-found` — the skill/tool ran but the signal isn't there. State the search performed. +- `unreachable` — the surface was not reachable from this agent context. State which. +- `unavailable` — no valid connector exists for this `evidenceType` (a gate-recorded gap). +- `out-of-scope` — the ask is `test_logs` or otherwise not the agent's to fulfill. + +An all-`unavailable` / all-`not-found` turn still resubmits — TFA decides how to +converge (best-effort, lower confidence) or what else to ask. The coordinator +does not pre-empt that decision. + +--- + +## Capability manifest (built once, at the gate) + +Rather than re-discover "is there a kibana skill?" on every ask across every +test, Gate Part A enumerates **and probe-validates** the client's connectors +**once** up front into a manifest (`lib/routing.mjs` → `buildManifest`). +`valid` maps to `available: true`; `invalid`/`absent` map to `available: false` +(a recorded gap): + +``` +{ github: {available: true, via: "gh"}, infra: {available: true, via: "kubectl"}, logs: {available: false}, ... } +``` + +- Every ask routes against this manifest — reproducible, no per-ask discovery. +- The gate summary **declares the gaps to the user** ("infra + metrics not + available") and the first turn declares them to TFA so it plans asks around + what's obtainable. +- Frozen at gate close. A skill appearing mid-run is not picked up until the next run. + +## Build-level evidence cache (compute once) + +"Diff since last green", "deploy timeline", and "PRs in the suspect window" are +properties of the **build**, not the test. The orchestrator computes the +last-green→this-build delta **once** (`lib/evidence-cache.mjs`), caches it by +`(repo, commit-range, evidenceType)`, and pre-seeds every coordinator with the +same grounded suspect window — collapsing N×M redundant git/infra calls to ~M and +front-loading the highest-signal evidence so many tests RESOLVE before any infra +ask fires. No "last green" (never-green suite) → fall back to a configured +baseline ref and note the weaker grounding in the turn digest (it lands in the +dashboard RCA). diff --git a/skills/rca-build/references/github-evidence.md b/skills/rca-build/references/github-evidence.md new file mode 100644 index 0000000..4b2c445 --- /dev/null +++ b/skills/rca-build/references/github-evidence.md @@ -0,0 +1,93 @@ +# GitHub evidence — what to gather, and how to rule a suspect OUT + +The worst automated-RCA outcome is **confidently blaming an innocent PR**. This +file is the contract for `product_code` / `deploy` / `ci` asks (the `github` +capability): the **exact** evidence to gather, and a **falsification protocol** +that tries to *disprove* each suspect before it enters `related_prs`. + +> We do **not** ship a GitHub forensics harness or MCP tool. We specify what's +> needed and use whatever the client already has — **GitHub MCP if available, +> else `gh`, else degrade** to an `unavailable` block. + +## Capability discovery (in order) + +1. **GitHub MCP** (`mcp__github__*`) — preferred for structured PR/diff/blame queries. +2. **`gh` CLI** — fall back for git-graph operations (`gh pr list --search`, + `gh api`, `merge-base`, ancestry) and anything the MCP doesn't cover. +3. **Neither** → emit an `unavailable` block for the ask (do not fabricate a PR). + +The gate records which is present **and probe-validated** (`gh auth status` / +GitHub MCP tools listed) in the capability manifest +(`capability: github → { available, via }`); route every github ask against it. + +## Application bugs REQUIRE a culprit-PR hunt (mandatory) + +Whenever TFA's working classification is **PRODUCT_BUG / application bug**, the +github connector is not optional evidence — it is the deliverable. The +coordinator MUST hunt the culprit PR: + +1. **Deploy timeline vs last-pass window** — what shipped to the run's env + between the last passing run and this failure. +2. **Changed paths vs failure signature** — intersect the window's PRs' changed + files with the failing file/function from the signature. +3. Run the falsification protocol below on each candidate. + +Feed the surviving PR **link(s)** to TFA in the turn message so the BrowserStack +agent populates `related_prs` in the dashboard RCA. **An application-bug RCA +with no GitHub PR link is INCOMPLETE**: keep digging on subsequent turns until +the turn cap. If still none, the turn must explicitly state +`no culprit PR identified after <what was searched: window, repos, paths>` and +the CSV row records the gap. Never fabricate a PR; if the github connector is +invalid/absent, the same explicit statement plus an `unavailable` block goes to +TFA (a gate-recorded gap). + +## Evidence each ask needs (be specific — no fishing) + +| Ask intent | Gather exactly | +|---|---| +| "Did `<X>` change since the last passing run?" | the diff of `<X>`'s file/function between the **baseline ref** (last-green, or the configured fallback) and the build's commit — not the whole repo diff | +| "Which PRs are suspect?" | PRs **merged in the window** `(baselineRef, build commit]` that **touch the failing code path** — intersect changed files with the failing file/function | +| "Who/what last changed the failing line?" | `blame` on the specific failing lines (from the test's `file_path` + the error) | +| "What shipped to the run's env before the failure?" | deploy timeline (`gh` releases/tags + the env's deploy record); compare deploy time vs. the run's `started_at` | +| "Did CI change?" | the workflow-file diff + recent `gh run` history for the failing job | + +Scope everything by the failing test's `file_path` + the error summary. The +build-level evidence (diff-since-last-green, PR window) is **pre-computed once** +and passed in — reuse it; do not re-fetch per test. + +## Falsification protocol — rule out, don't just rule in + +For **each** candidate suspect PR, try to **break** the hypothesis: + +1. **Path overlap.** Do the PR's changed hunks actually touch the failing code + path (the function/line in the stack)? No overlap → **ruled out**. +2. **Deployment-state guard.** Was the PR's code actually **live** in the run's + env at `started_at`? If it shipped *after* the failure window, or sits behind + an **OFF** flag, it could not have caused this failure → **ruled out**. +3. **Direction.** Does the change plausibly produce *this* error (e.g. a validator + tightened to reject the input the test sends)? If the change is unrelated to + the symptom → **weak**, mark accordingly. + +Feed **both supporting and disconfirming** evidence back to TFA. A suspect that +survives 1–3 is a real candidate; one that fails any is reported as ruled-out +(with the reason), **not** dropped silently. + +## The suspect packet (structured, not free text) + +Each surviving/ruled-out suspect is one structured block so `related_prs` +populates deterministically. **The canonical fillable format lives in +[`../templates/suspect-packet.md`](../templates/suspect-packet.md)** (fields: +pr, files, hunks, author, merged_at vs last_green vs started_at, verdict with +rule-out reason, link) — copy it, don't retype it. A worked example (supported ++ ruled-out side by side) is in +[`../examples/sample-run.md`](../examples/sample-run.md). + +Only `verdict: supported` suspects should end up in TFA's `related_prs`. Ruled-out +suspects stay in the thread as disconfirming evidence so TFA (and a human) can see +the elimination, not just the conclusion. + +## Digest discipline + +Same caps as `references/evidence-routing.md`: prefer a PR **link** over pasting a +diff; at most 1 hunk (3 hard) per `product_code` snippet; never paste a full diff. +The packet is *findings*, not the haystack. diff --git a/skills/rca-build/templates/evidence-block.md b/skills/rca-build/templates/evidence-block.md new file mode 100644 index 0000000..fa0bf49 --- /dev/null +++ b/skills/rca-build/templates/evidence-block.md @@ -0,0 +1,26 @@ +# Template — evidence block (one per fulfilled/unfulfilled ask) + +The unit of evidence sent back to TFA in a turn message. Rules, size caps and +the forbidden list: `../references/evidence-routing.md`. + +Fulfilled ask: + +``` +ASK: <verbatim `what` from the TfaAsk, ≤ 120 chars> +TYPE: <evidenceType> +FOUND: <yes | no | partial> +SUMMARY: <1–3 sentences — the finding, in the agent's words. ≤ 400 chars> +SNIPPET: + <the load-bearing excerpt only — see size caps. Omit if a LINK fully carries it.> +LINK: <permalink to the source — PR/commit/log-search/metrics panel/deploy record. Omit if N/A.> +``` + +Unfulfillable ask (report, don't drop — machine-generated for absent connectors +by `lib/loop.mjs` `unavailableBlock`): + +``` +ASK: <verbatim what> +TYPE: <evidenceType> +FOUND: no +SUMMARY: not-found | unreachable | unavailable | out-of-scope — <one line: what was checked or why blocked> +``` diff --git a/skills/rca-build/templates/gate-summary.md b/skills/rca-build/templates/gate-summary.md new file mode 100644 index 0000000..48ac6bf --- /dev/null +++ b/skills/rca-build/templates/gate-summary.md @@ -0,0 +1,25 @@ +# Template — gate summary (printed once, when THE GATE closes) + +One terse FYI before autonomous execution starts. Every intake field is tagged +`given | assumed | gap`; every connector `valid | invalid | absent`. After this +prints, the run never asks the user anything. + +If the product repo could not be corroborated against the failures AND no PRs +were supplied, the gate asks ONE question (interactive only) before printing +this summary. Headless skips it and prints `product repo: unknown (gap)`. + +``` +GATE CLOSED — capability manifest: + github ✅ valid (gh, authed) · infra ✅ valid (via <kubectl ctx …, docker, ecs, …>) · logs ❌ absent · metrics ❌ absent + +Intake: + build id: <id> (given) + product repo: <org/repo> (assumed — corroborated vs failures | asked — human answered | unknown (gap)) + automation repo: <org/repo> (assumed — cwd holds the tests) + working branch: <branch> (assumed — current branch) + default branch: <branch> (assumed — origin HEAD) + PRs in play: <#123, #456 | none> (given | gap) + +Gaps declared to TFA: <logs, metrics | none> +Proceeding autonomously: discovery → clustering → fan-out (concurrency <N>, turn-cap <M>). +``` diff --git a/skills/rca-build/templates/suspect-packet.md b/skills/rca-build/templates/suspect-packet.md new file mode 100644 index 0000000..676fcf4 --- /dev/null +++ b/skills/rca-build/templates/suspect-packet.md @@ -0,0 +1,22 @@ +# Template — SUSPECT packet (one block per candidate PR) + +Fill one block per suspect, supported **and** ruled-out (elimination is evidence +too). Only `verdict: supported` suspects may feed `related_prs`. Guidance + +falsification protocol: `../references/github-evidence.md`. + +``` +SUSPECT: + pr: <#number> + files: <changed files overlapping the failing path> + hunks: <the 1-3 load-bearing changed hunks — see digest size caps> + author: <login> + merged_at: <ts> vs last_green: <ts> vs started_at: <ts> + verdict: supported | ruled-out (<no-path-overlap | shipped-after | behind-off-flag | unrelated>) + link: <PR permalink> +``` + +If the hunt ends empty after a real search (never fabricate): + +``` +no culprit PR identified after <what was searched: window, repos, paths> +``` diff --git a/tests/conformance.test.mjs b/tests/conformance.test.mjs new file mode 100644 index 0000000..3dde60b --- /dev/null +++ b/tests/conformance.test.mjs @@ -0,0 +1,127 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; +import { runRcaLoop, replaySubmit } from "../lib/loop.mjs"; + +const here = dirname(fileURLToPath(import.meta.url)); +const load = (name) => + JSON.parse(readFileSync(join(here, "fixtures", "recorded-turns", name), "utf8")); + +const CONFIG = { + turnCap: 6, + evidenceRouting: { + test_logs: { owner: "tfa", skip: true }, + product_code: { capability: "github" }, + other: { capability: "other" }, + }, +}; +const GITHUB_AVAILABLE = { github: { available: true, via: "gh" } }; + +// A coordinator gather() stub: returns a one-line digest block. +const gather = async (g) => `ASK: ${g.ask.what}\nTYPE: ${g.evidenceType}\nFOUND: yes\nSUMMARY: stub`; + +test("resolved fixture: NEEDS_INFO → evidence → RESOLVED, trimmed glimpse captured, test_logs skipped", async () => { + const fx = load("resolved.json"); + const result = await runRcaLoop({ + testRunId: fx.testRunId, + firstMessage: "Error: empty buildName", + submit: replaySubmit(fx.turns), + config: CONFIG, + manifest: GITHUB_AVAILABLE, + gather, + }); + assert.equal(result.status, "RESOLVED"); + assert.match(result.root_cause, /#7421/); + assert.ok(result.root_cause.length <= 220); // glimpse root_cause is trimmed server-side + assert.equal(result.failure_type, "product_regression"); + assert.deepEqual(result.related_prs, ["#7421"]); + assert.match(result.view_rca, /^https:\/\/automation\.browserstack\.com/); + assert.deepEqual(result.asks_fulfilled, ["product_code"]); + assert.deepEqual(result.asks_skipped, ["test_logs"]); // TFA-owned, never gathered + assert.equal(result.turns_used, 2); + assert.equal(result.threadId, "thr-39"); +}); + +test("pending fixture: soft-PENDING (trimmed: status/threadId/turnId) ends with turnId, no re-poll", async () => { + const fx = load("pending.json"); + let calls = 0; + const counting = async (args) => { + calls++; + return replaySubmit(fx.turns)(args); + }; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: counting, + config: CONFIG, + }); + assert.equal(result.status, "PENDING"); + assert.equal(result.turnId, "turn-81-1"); + assert.equal(result.threadId, "thr-81"); + assert.equal(calls, 1); // ended immediately, did not poll again +}); + +test("turn-cap fixture: ends PENDING(turn-cap) at the cap, never a 7th submit", async () => { + const fx = load("turn-cap.json"); + let submits = 0; + const counting = async (args) => { + submits++; + return replaySubmit(fx.turns)(args); + }; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: counting, + config: CONFIG, + manifest: GITHUB_AVAILABLE, + gather, + }); + assert.equal(result.status, "PENDING"); + assert.equal(result.root_cause, "turn-cap"); + assert.equal(submits, 6); // capped at turnCap, never 7 +}); + +test("degraded path: no connector → gap degrades to unavailable (never a prompt), still terminal", async () => { + // Same resolved fixture, but the client has NO github connector. The loop is + // autonomous: the gap becomes an `unavailable` block and it still RESOLVEs. + const fx = load("resolved.json"); + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: replaySubmit(fx.turns), + config: CONFIG, + manifest: {}, // nothing valid at the gate + }); + assert.equal(result.status, "RESOLVED"); + assert.deepEqual(result.asks_unavailable, ["product_code"]); + assert.deepEqual(result.asks_fulfilled, []); +}); + +test("unavailable block names the missing connector in the resubmitted message", async () => { + const fx = load("resolved.json"); + const messages = []; + const recording = (inner) => async (args) => { + messages.push(args.message); + return inner(args); + }; + await runRcaLoop({ + testRunId: fx.testRunId, + submit: recording(replaySubmit(fx.turns)), + config: CONFIG, + manifest: {}, + }); + assert.match(messages[1], /unavailable — no github connector/); +}); + +test("no testRunId → failed block, tool never called", async () => { + let called = false; + const result = await runRcaLoop({ + testRunId: undefined, + submit: async () => { + called = true; + return {}; + }, + config: CONFIG, + }); + assert.equal(result.status, "failed"); + assert.equal(called, false); +}); diff --git a/tests/coverage-glimpse.test.mjs b/tests/coverage-glimpse.test.mjs new file mode 100644 index 0000000..3bc0e48 --- /dev/null +++ b/tests/coverage-glimpse.test.mjs @@ -0,0 +1,93 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { coverageStamp, classifyCoverage } from "../lib/coverage.mjs"; +import { renderGlimpse } from "../lib/glimpse.mjs"; + +// ---- coverage stamp -------------------------------------------------------- + +test("full coverage keeps TFA confidence", () => { + const s = coverageStamp({ + asksFulfilled: ["product_code"], + asksUnavailable: [], + tfaConfidence: "high", + }); + assert.equal(s.coverage, "full"); + assert.equal(s.band, "high"); +}); + +test("partial coverage caps a high TFA confidence at medium", () => { + const s = coverageStamp({ + asksFulfilled: ["product_code"], + asksUnavailable: ["kibana"], + tfaConfidence: "high", + }); + assert.equal(s.coverage, "partial"); + assert.equal(s.band, "medium"); + assert.deepEqual(s.unavailable, ["kibana"]); +}); + +test("thin coverage (nothing fulfilled, gaps) caps at low", () => { + const s = coverageStamp({ + asksFulfilled: [], + asksUnavailable: ["infra", "metrics"], + tfaConfidence: "high", + }); + assert.equal(s.coverage, "thin"); + assert.equal(s.band, "low"); +}); + +test("unknown TFA confidence floors to low even at full coverage", () => { + const s = coverageStamp({ asksFulfilled: [], asksUnavailable: [], tfaConfidence: "unknown" }); + assert.equal(s.coverage, "full"); + assert.equal(s.band, "low"); +}); + +test("classifyCoverage dedupes and handles empties", () => { + assert.equal(classifyCoverage(["a", "a"], []), "full"); + assert.equal(classifyCoverage([], ["x"]), "thin"); +}); + +// ---- glimpse (the ONLY in-client output — no local report) ------------------ + +test("empty batch renders a valid glimpse, no crash", () => { + const txt = renderGlimpse([], { buildId: "b1" }); + assert.match(txt, /No failed tests analyzed/); +}); + +test("glimpse is a completion notice with counts — NO per-test detail", () => { + const rows = [ + { + testRunId: "101", + cluster_id: "c1", + rca_done: "resolved", + confidence: "high", + root_cause: "PR #7421 tightened validator", + related_prs: "#7421", + }, + { testRunId: "102", cluster_id: "c1", rca_done: "pending-resume" }, + { testRunId: "103", cluster_id: "c2", rca_done: "failed" }, + ]; + const txt = renderGlimpse(rows, { buildId: "b1" }); + assert.match(txt, /RCA analysis complete — build b1/); + assert.match(txt, /3 test\(s\)/); + assert.match(txt, /1 resolved/); + assert.match(txt, /1 pending/); // pending-resume buckets to "pending" + assert.match(txt, /1 failed/); + // The trim: no per-test lines, no root cause, no PRs, no testRunIds leak. + assert.doesNotMatch(txt, /7421|PR #|→|101|102|103|c1|c2/); +}); + +test("glimpse never leaks a verbose root_cause, however long", () => { + const rows = [ + { + testRunId: "1", + cluster_id: "solo-1", + rca_done: "resolved", + confidence: "medium", + root_cause: `line one\nline two ${"x".repeat(200)}`, + }, + ]; + const txt = renderGlimpse(rows); + assert.doesNotMatch(txt, /line one|line two|xxxx/); // cause stays out of Claude + assert.match(txt, /1 test\(s\) · 1 resolved/); +}); diff --git a/tests/csv-state.test.mjs b/tests/csv-state.test.mjs new file mode 100644 index 0000000..3cbc83a --- /dev/null +++ b/tests/csv-state.test.mjs @@ -0,0 +1,187 @@ +import { test, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + csvPathFor, + seed, + readRows, + claim, + heartbeat, + flip, + reaper, + pendingRows, + PENDING, +} from "../lib/csv-state.mjs"; + +let dir; +let csv; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "rca-csv-")); + csv = join(dir, "state.csv"); +}); +afterEach(() => rmSync(dir, { recursive: true, force: true })); + +const TESTS = [ + { + test_id: 101, + test_name: "login", + failure: { category: "Assertion", error_summary: "expected 200", file_path: "a.rb" }, + }, + { test_id: 102, test_name: "checkout", failure: { category: "Timeout" } }, +]; + +test("seed writes one pending row per test with signature columns", () => { + const rows = seed(csv, "build-1", TESTS); + assert.equal(rows.length, 2); + assert.ok(rows.every((r) => r.rca_done === PENDING)); + const login = rows.find((r) => r.testRunId === "101"); + assert.equal(login.failure_category, "Assertion"); + assert.equal(login.error_summary, "expected 200"); + assert.equal(login.buildId, "build-1"); +}); + +test("seed is idempotent — no duplicate rows on re-seed", () => { + seed(csv, "build-1", TESTS); + const rows = seed(csv, "build-1", TESTS); + assert.equal(rows.length, 2); +}); + +test("seed preserves a terminal row on re-seed", () => { + seed(csv, "build-1", TESTS); + flip(csv, 101, { rca_done: "resolved", root_cause: "bad PR" }, 1000); + seed(csv, "build-1", TESTS); + const login = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(login.rca_done, "resolved"); + assert.equal(login.root_cause, "bad PR"); +}); + +test("claim sets the worker; a second worker is refused", () => { + seed(csv, "build-1", TESTS); + assert.equal(claim(csv, 101, "w1", 1000), true); + assert.equal(claim(csv, 101, "w2", 1000), false); + const row = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(row.in_flight_worker, "w1"); +}); + +test("heartbeat updates ts only for the owning worker", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + assert.equal(heartbeat(csv, 101, "w1", 2000), true); + assert.equal(heartbeat(csv, 101, "w2", 3000), false); + assert.equal(readRows(csv).find((r) => r.testRunId === "101").heartbeat_ts, "2000"); +}); + +test("flip records terminal fields, joins related_prs, clears the claim", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + flip( + csv, + 101, + { rca_done: "resolved", root_cause: "PR #7421", related_prs: ["#7421", "#7430"], confidence: "high" }, + 5000, + ); + const row = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(row.rca_done, "resolved"); + assert.equal(row.related_prs, "#7421; #7430"); + assert.equal(row.confidence, "high"); + assert.equal(row.in_flight_worker, ""); + assert.equal(row.timestamp, "5000"); +}); + +test("reaper reclaims only stale in-flight rows", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); // stale + claim(csv, 102, "w2", 9000); // fresh + const ttl = 600; // seconds + const now = 1000 + ttl * 1000 + 1; // just past TTL for w1, fresh for w2 + const reclaimed = reaper(csv, ttl, now); + assert.deepEqual(reclaimed, ["101"]); + const rows = readRows(csv); + assert.equal(rows.find((r) => r.testRunId === "101").in_flight_worker, ""); + assert.equal(rows.find((r) => r.testRunId === "101").rca_done, PENDING); + assert.equal(rows.find((r) => r.testRunId === "102").in_flight_worker, "w2"); +}); + +test("reaper leaves terminal rows alone even if in_flight lingered", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + flip(csv, 101, { rca_done: "resolved" }, 2000); // flip clears in_flight + const reclaimed = reaper(csv, 600, 10_000_000); + assert.deepEqual(reclaimed, []); +}); + +test("pendingRows returns only pending work", () => { + seed(csv, "build-1", TESTS); + flip(csv, 101, { rca_done: "resolved" }, 1000); + const pend = pendingRows(csv); + assert.equal(pend.length, 1); + assert.equal(pend[0].testRunId, "102"); +}); + +test("flip rejects a missing/non-terminal rca_done without mutating the row", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + // missing rca_done + assert.equal(flip(csv, 101, { root_cause: "x" }, 2000), false); + // invalid rca_done + assert.equal(flip(csv, 101, { rca_done: "weird" }, 2000), false); + const row = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(row.rca_done, PENDING); // not reverted to claimable-pending silently + assert.equal(row.in_flight_worker, "w1"); // claim intact — bug surfaces, no clobber + assert.equal(row.root_cause, ""); // nothing written +}); + +test("pending-resume is resumable: not terminal, listed, and re-claimable", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + flip(csv, 101, { rca_done: "pending-resume", threadId: "thr-1", turnId: "t-1" }, 2000); + const row = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(row.in_flight_worker, ""); // this attempt released the claim + assert.equal(row.threadId, "thr-1"); // resume handles retained + assert.equal(row.turnId, "t-1"); + // appears in the fan-out work-list and can be claimed by the resume pass + assert.ok(pendingRows(csv).some((r) => r.testRunId === "101")); + assert.equal(claim(csv, 101, "w2", 3000), true); +}); + +test("reaper ignores pending-resume rows (not in flight)", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + flip(csv, 101, { rca_done: "pending-resume" }, 2000); + assert.deepEqual(reaper(csv, 600, 10_000_000), []); +}); + +test("CSV codec round-trips fields with commas, quotes, newlines", () => { + seed(csv, "build-1", [{ test_id: 200, test_name: "weird" }]); + flip( + csv, + 200, + { rca_done: "resolved", root_cause: 'Failed: "x", got <y>\nsecond line' }, + 1000, + ); + const row = readRows(csv).find((r) => r.testRunId === "200"); + assert.equal(row.root_cause, 'Failed: "x", got <y>\nsecond line'); +}); + +test("csvPathFor: build id is in the filename, default dir is OS temp", () => { + const p = csvPathFor("abc123XYZ"); + assert.ok(p.startsWith(join(tmpdir(), "bstack-rca"))); + assert.ok(p.endsWith("rca-state.abc123XYZ.csv")); +}); + +test("csvPathFor: different builds never share a path", () => { + assert.notEqual(csvPathFor("build-A"), csvPathFor("build-B")); +}); + +test("csvPathFor: sanitizes hostile ids and handles empty", () => { + assert.ok(csvPathFor("../../etc/passwd").endsWith("rca-state..._.._etc_passwd.csv")); + assert.ok(csvPathFor("").endsWith("rca-state.unknown-build.csv")); +}); + +test("csvPathFor: stateDir override wins over temp", () => { + const p = csvPathFor("b1", "/ci/artifacts"); + assert.equal(p, join("/ci/artifacts", "rca-state.b1.csv")); +}); diff --git a/tests/evidence.test.mjs b/tests/evidence.test.mjs new file mode 100644 index 0000000..041c694 --- /dev/null +++ b/tests/evidence.test.mjs @@ -0,0 +1,77 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { buildManifest, unavailableCapabilities } from "../lib/routing.mjs"; +import { makeEvidenceCache, resolveBaseline } from "../lib/evidence-cache.mjs"; + +const CONFIG = { + evidenceRouting: { + test_logs: { owner: "tfa", skip: true }, + product_code: { capability: "github" }, + deploy: { capability: "github" }, + infra: { capability: "infra" }, + k8s: { capability: "infra" }, + metrics: { capability: "metrics" }, + other: { capability: "other" }, + }, +}; + +test("buildManifest marks discovered capabilities available with via", () => { + const manifest = buildManifest(CONFIG, [ + { capability: "github", via: "github-mcp" }, + ]); + assert.equal(manifest.github.available, true); + assert.equal(manifest.github.via, "github-mcp"); + assert.equal(manifest.infra.available, false); +}); + +test("buildManifest excludes the TFA-owned test_logs capability", () => { + const manifest = buildManifest(CONFIG, []); + assert.ok(!("undefined" in manifest)); + assert.ok(!Object.keys(manifest).includes("test_logs")); +}); + +test("buildManifest dedupes capabilities shared by multiple evidence types", () => { + // product_code + deploy both map to github → one manifest entry + const manifest = buildManifest(CONFIG, [{ capability: "github" }]); + assert.equal(Object.keys(manifest).filter((k) => k === "github").length, 1); +}); + +test("unavailableCapabilities lists what the client can't get", () => { + const manifest = buildManifest(CONFIG, [{ capability: "github" }]); + const unavailable = unavailableCapabilities(manifest).sort(); + assert.deepEqual(unavailable, ["infra", "metrics", "other"]); +}); + +test("evidence cache computes once and reuses across calls", async () => { + const cache = makeEvidenceCache(); + let calls = 0; + const fn = async () => { + calls++; + return { prs: ["#1"] }; + }; + const a = await cache.compute("repo", "abc..def", "deploy", fn); + const b = await cache.compute("repo", "abc..def", "deploy", fn); + assert.equal(calls, 1); + assert.deepEqual(a, b); + assert.equal(cache.size(), 1); +}); + +test("evidence cache key distinguishes commit ranges", async () => { + const cache = makeEvidenceCache(); + let calls = 0; + const fn = async () => ++calls; + await cache.compute("repo", "r1", "deploy", fn); + await cache.compute("repo", "r2", "deploy", fn); + assert.equal(calls, 2); +}); + +test("resolveBaseline uses last-green when present, else flags fallback", () => { + assert.deepEqual(resolveBaseline("v1.2.3", "main"), { + ref: "v1.2.3", + isFallback: false, + }); + assert.deepEqual(resolveBaseline(null, "main"), { + ref: "main", + isFallback: true, + }); +}); diff --git a/tests/fixtures/recorded-turns/pending.json b/tests/fixtures/recorded-turns/pending.json new file mode 100644 index 0000000..2e73ce7 --- /dev/null +++ b/tests/fixtures/recorded-turns/pending.json @@ -0,0 +1,11 @@ +{ + "name": "soft-pending — resumable (trimmed shape: status/threadId/turnId only)", + "testRunId": 81, + "turns": [ + { + "status": "PENDING", + "threadId": "thr-81", + "turnId": "turn-81-1" + } + ] +} diff --git a/tests/fixtures/recorded-turns/resolved.json b/tests/fixtures/recorded-turns/resolved.json new file mode 100644 index 0000000..a7964b8 --- /dev/null +++ b/tests/fixtures/recorded-turns/resolved.json @@ -0,0 +1,41 @@ +{ + "name": "needs_info → evidence → resolved (trimmed terminal glimpse)", + "testRunId": 39, + "turns": [ + { + "status": "NEEDS_INFO", + "confidence": "low", + "threadId": "thr-39", + "questions": [ + "Did the buildName validator change?" + ], + "asks": [ + { + "what": "Did request-validation on POST /builds change since last green?", + "why": "the failing test posts an empty buildName", + "evidenceType": "product_code", + "priority": "high" + }, + { + "what": "Full run logs for test 39", + "why": "to read the failure", + "evidenceType": "test_logs", + "priority": "high" + } + ] + }, + { + "status": "RESOLVED", + "confidence": "high", + "threadId": "thr-39", + "glimpse": { + "root_cause": "PR #7421 tightened the buildName validator to reject empty strings", + "failure_type": "product_regression", + "related_prs": [ + "#7421" + ] + }, + "viewRca": "https://automation.browserstack.com — open the build's AI report (tab=ai_report, subTab=aitfa) to view the full RCA" + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/recorded-turns/turn-cap.json b/tests/fixtures/recorded-turns/turn-cap.json new file mode 100644 index 0000000..4638b61 --- /dev/null +++ b/tests/fixtures/recorded-turns/turn-cap.json @@ -0,0 +1,12 @@ +{ + "name": "turn-cap — never resolves", + "testRunId": 99, + "turns": [ + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] } + ] +} diff --git a/tests/routing.test.mjs b/tests/routing.test.mjs new file mode 100644 index 0000000..6ebdb9b --- /dev/null +++ b/tests/routing.test.mjs @@ -0,0 +1,81 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { routeAsk, routeAsks, orderAsks, TEST_LOGS } from "../lib/routing.mjs"; + +const CONFIG = { + evidenceRouting: { + test_logs: { owner: "tfa", skip: true }, + product_code: { capability: "github", discoveryHints: ["github-mcp", "gh"] }, + infra: { capability: "infra", discoveryHints: [] }, + k8s: { capability: "infra", discoveryHints: [] }, + other: { capability: "other", discoveryHints: [] }, + }, +}; + +test("test_logs is always skipped (TFA-owned)", () => { + const r = routeAsk({ evidenceType: TEST_LOGS, priority: "high" }, CONFIG, { + github: { available: true }, + }); + assert.equal(r.action, "skip"); + assert.equal(r.reason, "tfa-owned"); +}); + +test("available capability → gather, carrying via", () => { + const r = routeAsk({ evidenceType: "product_code", priority: "high" }, CONFIG, { + github: { available: true, via: "github-mcp" }, + }); + assert.equal(r.action, "gather"); + assert.equal(r.capability, "github"); + assert.equal(r.via, "github-mcp"); +}); + +test("unavailable capability → gap, carrying discovery hints", () => { + const r = routeAsk({ evidenceType: "k8s", priority: "medium" }, CONFIG, { + infra: { available: false }, + }); + assert.equal(r.action, "gap"); + assert.equal(r.capability, "infra"); + assert.equal(r.reason, "no-capability"); +}); + +test("capability absent from manifest entirely → gap", () => { + const r = routeAsk({ evidenceType: "k8s", priority: "low" }, CONFIG, {}); + assert.equal(r.action, "gap"); +}); + +test("unknown evidenceType falls back to the 'other' entry", () => { + const r = routeAsk({ evidenceType: "weird", priority: "low" }, CONFIG, { + other: { available: true, via: "best-effort" }, + }); + assert.equal(r.action, "gather"); + assert.equal(r.capability, "other"); +}); + +test("orderAsks sorts high → medium → low, unknown last", () => { + const ordered = orderAsks([ + { what: "c", priority: "low" }, + { what: "a", priority: "high" }, + { what: "d", priority: undefined }, + { what: "b", priority: "medium" }, + ]); + assert.deepEqual( + ordered.map((a) => a.what), + ["a", "b", "c", "d"], + ); +}); + +test("routeAsks buckets a mixed turn in priority order", () => { + const buckets = routeAsks( + [ + { evidenceType: "k8s", priority: "low" }, + { evidenceType: "test_logs", priority: "high" }, + { evidenceType: "product_code", priority: "high" }, + ], + CONFIG, + { github: { available: true, via: "gh" } }, + ); + assert.equal(buckets.skip.length, 1); + assert.equal(buckets.gather.length, 1); + assert.equal(buckets.gap.length, 1); + assert.equal(buckets.gather[0].evidenceType, "product_code"); +}); diff --git a/tests/signature.test.mjs b/tests/signature.test.mjs new file mode 100644 index 0000000..f721167 --- /dev/null +++ b/tests/signature.test.mjs @@ -0,0 +1,79 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + normalize, + computeSignature, + selectRepresentative, + clusterRows, +} from "../lib/signature.mjs"; + +function row(id, extra = {}) { + return { + testRunId: String(id), + failure_category: "Assertion", + error_summary: "expected 200 but got 500", + file_path: "spec/login.rb", + is_flaky: "false", + ...extra, + }; +} + +test("normalize folds timestamps, uuids, hex, line:col, and numbers", () => { + assert.equal(normalize("Error at line :42:7"), "error at line :<line>"); + assert.equal(normalize("got 500 at 0xAF3"), "got <n> at <hex>"); + assert.equal( + normalize("failed 2026-06-23T10:00:00Z"), + "failed <ts>", + ); +}); + +test("identical category+error+path → same cluster", () => { + const { clusters } = clusterRows([row(1), row(2)]); + assert.equal(clusters.length, 1); + assert.equal(clusters[0].members.length, 2); +}); + +test("numbers in the error are folded so siblings still cluster", () => { + const a = row(1, { error_summary: "timeout after 3000ms on node-7" }); + const b = row(2, { error_summary: "timeout after 5000ms on node-2" }); + assert.equal(computeSignature(a), computeSignature(b)); + const { clusters } = clusterRows([a, b]); + assert.equal(clusters.length, 1); +}); + +test("distinct failures → distinct clusters", () => { + const a = row(1, { error_summary: "null pointer in Foo" }); + const b = row(2, { error_summary: "connection refused" }); + const { clusters } = clusterRows([a, b]); + assert.equal(clusters.length, 2); +}); + +test("rows with no signal become their own singletons (no catch-all merge)", () => { + const a = { testRunId: "1", failure_category: "", error_summary: "", file_path: "" }; + const b = { testRunId: "2", failure_category: "", error_summary: "", file_path: "" }; + const { clusters } = clusterRows([a, b]); + assert.equal(clusters.length, 2); + assert.ok(clusters.every((c) => c.cluster_id.startsWith("solo-"))); +}); + +test("singleton cluster has a representative and no siblings", () => { + const { clusters } = clusterRows([row(1)]); + assert.equal(clusters[0].siblings.length, 0); + assert.equal(clusters[0].representative.testRunId, "1"); +}); + +test("representative is deterministic: non-flaky, then smallest testRunId", () => { + const members = [ + row(5, { is_flaky: "true" }), + row(9, { is_flaky: "false" }), + row(7, { is_flaky: "false" }), + ]; + assert.equal(selectRepresentative(members).testRunId, "7"); +}); + +test("clusterRows stamps cluster_id onto every row", () => { + const rows = [row(1), row(2, { error_summary: "different" })]; + clusterRows(rows); + assert.ok(rows.every((r) => r.cluster_id)); + assert.notEqual(rows[0].cluster_id, rows[1].cluster_id); +}); diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs new file mode 100644 index 0000000..553b94a --- /dev/null +++ b/workflows/rca-batch.mjs @@ -0,0 +1,135 @@ +export const meta = { + name: "rca-batch", + description: + "Drive autonomous collaborative RCA over all failed tests of a build: cluster representatives run the full loop, siblings one-turn-confirm, ~5 concurrent. Never prompts a user.", + phases: [ + { title: "Representatives", detail: "full multi-turn RCA per cluster" }, + { title: "Siblings", detail: "one-turn confirm against own logs" }, + ], +}; + +// The /rca-build batch orchestration (fully autonomous — the gate closed before +// this runs; nothing here ever asks the user). This is a dynamic-workflow +// script: it runs in the Workflow sandbox (no filesystem, no Date.now/ +// Math.random, agent()/pipeline() as globals). It therefore does NO state I/O +// itself — the orchestrator seeds the CSV, clusters, and builds the validated +// manifest at the gate and passes the work-list via `args`; each dispatched +// `ai-tfa-coordinator` agent (which HAS tool access) claims + flips its own CSV +// row eagerly (WAL); this script orchestrates concurrency and returns the +// structured results for reconciliation. The final glimpse + triggerRcaReport +// step happens back in the orchestrator (SKILL.md Step 6). +// +// args shape: +// { +// csvPath, buildId, +// manifest: { capability: { available, via } }, +// buildEvidence: { baselineRef, suspectWindow, ... }, // pre-computed once +// clusters: [ +// { cluster_id, representative: { testRunId, testName, error_summary }, +// siblings: [ { testRunId, testName, error_summary } ] } +// ] +// } + +const RCA_SCHEMA = { + type: "object", + required: ["testRunId", "status"], + properties: { + testRunId: { type: "string" }, + status: { enum: ["RESOLVED", "PENDING", "failed"] }, + confidence: { enum: ["high", "medium", "low", "unknown"] }, + root_cause: { type: "string" }, + failure_type: { type: "string" }, + view_rca: { type: "string" }, + related_prs: { type: "array", items: { type: "string" } }, + suspect_signals: { type: "array", items: { type: "string" } }, + threadId: { type: "string" }, + turnId: { type: "string" }, + turns_used: { type: "number" }, + asks_fulfilled: { type: "array", items: { type: "string" } }, + asks_skipped: { type: "array", items: { type: "string" } }, + asks_unavailable: { type: "array", items: { type: "string" } }, + cluster_id: { type: "string" }, + }, + additionalProperties: true, +}; + +const ctx = args ?? {}; +const clusters = ctx.clusters ?? []; +const shared = [ + `CSV state file: ${ctx.csvPath}`, + `Capability manifest: ${JSON.stringify(ctx.manifest ?? {})}`, + `Build-level evidence (pre-computed once, reuse — do not re-fetch): ${JSON.stringify(ctx.buildEvidence ?? {})}`, + `Autonomous run — on an evidence gap with no valid connector, report "unavailable" back to TFA (NEVER prompt a user). Best-effort finalize.`, + `PRODUCT_BUG / application-bug mandate: hunt the culprit PR via the github connector (deploy timeline vs last-pass window, changed paths vs failure signature) and feed the PR link(s) to TFA so related_prs populates. No PR after digging to the turn cap → state explicitly "no culprit PR identified after <what was searched>" so the CSV row records the gap.`, + `Persist eagerly to the CSV: claim your row before turn 1, flip it on terminal (lib/csv-state.mjs).`, +].join("\n"); + +function repPrompt(cluster) { + const r = cluster.representative; + return [ + `You are the ai-tfa-coordinator for cluster ${cluster.cluster_id}.`, + `Run the FULL collaborative RCA loop for the representative test.`, + `testRunId=${r.testRunId} testName=${r.testName ?? ""}`, + `error_digest: ${r.error_summary ?? "(none)"}`, + shared, + `Return the structured RCA_OUTPUT for this test.`, + ].join("\n"); +} + +function siblingPrompt(sibling, repResult, cluster) { + return [ + `You are the ai-tfa-coordinator for a SIBLING of cluster ${cluster.cluster_id}.`, + `Pre-seed: the representative resolved as:`, + ` root_cause: ${repResult?.root_cause ?? "(representative did not resolve)"}`, + ` related_prs: ${JSON.stringify(repResult?.related_prs ?? [])}`, + `State this hypothesis on turn 1 and ask TFA to CONFIRM it against THIS test's own logs.`, + `If TFA confirms in one turn → done. If it does NOT (NEEDS_INFO), fall back to the full loop — never blindly inherit.`, + `testRunId=${sibling.testRunId} testName=${sibling.testName ?? ""}`, + `error_digest: ${sibling.error_summary ?? "(none)"}`, + shared, + `Return the structured RCA_OUTPUT for this test.`, + ].join("\n"); +} + +log(`Batch: ${clusters.length} cluster(s) over build ${ctx.buildId ?? "?"}`); + +// Pipeline: each cluster flows representative → siblings independently (no barrier +// between stages), so a small cluster's siblings confirm while a big cluster's +// representative is still looping. Concurrency is bounded by the workflow runtime +// (~min(16, cores-2)); config.concurrency (5) is the intended soft target. +const results = await pipeline( + clusters, + (cluster) => + agent(repPrompt(cluster), { + label: `rep:${cluster.representative.testRunId}`, + phase: "Representatives", + agentType: "tfa-rca:ai-tfa-coordinator", + schema: RCA_SCHEMA, + }).then((rca) => ({ cluster, rca })), + ({ cluster, rca }) => + parallel( + (cluster.siblings ?? []).map((sib) => () => + agent(siblingPrompt(sib, rca, cluster), { + label: `sib:${sib.testRunId}`, + phase: "Siblings", + agentType: "tfa-rca:ai-tfa-coordinator", + schema: RCA_SCHEMA, + }), + ), + ).then((sibs) => ({ + cluster_id: cluster.cluster_id, + representative: rca, + siblings: sibs.filter(Boolean), + })), +); + +const flat = results.filter(Boolean); +const all = flat.flatMap((r) => [r.representative, ...(r.siblings ?? [])]).filter(Boolean); +const byStatus = all.reduce((acc, r) => { + acc[r.status] = (acc[r.status] ?? 0) + 1; + return acc; +}, {}); + +log(`Batch complete: ${all.length} test(s) — ${JSON.stringify(byStatus)}`); + +return { clusters: flat.length, tests: all.length, byStatus, results: flat };