feat: AI failure explanation (explainFailure)

CHANGELOG.md

@@ -15,6 +15,9 @@ All notable changes to this project are documented here. This project follows
   fetch (`createProvider`, bring your own key, OpenAI/Anthropic/custom baseURL)
   and `aiGenerateTests` to draft a suite from an endpoint or sample response.
   New CLI command `two-go ai gen <url|file> [-o out]`.
+- **AI failure explanation** (`explainFailure`): send a failed assertion plus
+  the request and response context to an LLM and get a likely cause and a
+  suggested fix. Advisory only, it never changes pass or fail.
 
 ## [0.4.0]
 

README.md

@@ -630,6 +630,20 @@ like any other test. Treat it as a first draft and tighten the checks. The
 default models are `gpt-5.3` for OpenAI and `claude-opus-4-8` for Anthropic, and
 you can override either with `{ model }`.
 
+When a test fails you can also ask the model what probably went wrong. This is
+advisory, it runs after the failure and never changes pass or fail.
+
+```js
+import { explainFailure } from "two-go/ai";
+
+try {
+  await api.get("/users").expectStatus(200);
+} catch (err) {
+  const why = await explainFailure(err, { response: err.response, provider: "openai" });
+  console.log(why); // likely cause plus a suggested fix
+}
+```
+
 ## TypeScript
 
 Types are written by hand and shipped with the package, so you get

src/ai/explain.d.ts

@@ -0,0 +1,12 @@
+// Type declarations for AI failure explanation.
+import type { Provider, ProviderOptions } from "./provider.js";
+
+export interface ExplainOptions extends Omit<ProviderOptions, "provider"> {
+  response?: unknown;
+  request?: unknown;
+  maxTokens?: number;
+  provider?: Provider | ProviderOptions["provider"];
+}
+
+/** Ask an LLM for the likely cause of a failed assertion and a suggested fix. */
+export declare function explainFailure(error: unknown, options?: ExplainOptions): Promise<string>;

src/ai/explain.js

@@ -0,0 +1,72 @@
+// Explain a failed assertion with an LLM. This is advisory only: it runs after
+// a test already failed and never changes pass or fail. It takes the error plus
+// whatever request and response context you have and asks for a short, specific
+// likely cause and a concrete fix.
+
+import { createProvider } from "./provider.js";
+
+const SYSTEM = [
+  "You are a senior API engineer triaging a failed test.",
+  "Given the failed assertion and the request and response, explain the most",
+  "likely cause in two to four sentences, then suggest one concrete fix.",
+  "Be specific. Do not invent details that are not present in the input.",
+].join("\n");
+
+function truncate(value, max = 2000) {
+  const s = value == null ? "" : String(value);
+  return s.length > max ? s.slice(0, max) + "... (truncated)" : s;
+}
+
+function buildPrompt(error, options) {
+  const lines = [];
+  lines.push("Failed assertion:");
+  lines.push(error && error.message ? error.message : String(error));
+  if (error && error.expected !== undefined) {
+    lines.push("Expected: " + truncate(JSON.stringify(error.expected), 500));
+  }
+  if (error && error.actual !== undefined) {
+    lines.push("Actual: " + truncate(JSON.stringify(error.actual), 500));
+  }
+
+  const res = options.response;
+  if (res) {
+    lines.push("");
+    lines.push("Response:");
+    const line = `${res.method || ""} ${res.url || ""}`.trim();
+    if (line) lines.push(line);
+    if (res.status != null) {
+      lines.push("Status: " + res.status + (res.statusText ? " " + res.statusText : ""));
+    }
+    if (res.time != null) lines.push("Time: " + res.time + "ms");
+    if (res.headers) lines.push("Headers: " + truncate(JSON.stringify(res.headers), 800));
+    const body = res.text != null ? res.text : res.body !== undefined ? JSON.stringify(res.body) : "";
+    if (body) lines.push("Body: " + truncate(body));
+  }
+
+  const req = options.request;
+  if (req) {
+    lines.push("");
+    lines.push("Request:");
+    const headers = req._headers || req.headers;
+    if (headers) lines.push("Sent headers: " + truncate(JSON.stringify(headers), 500));
+    const reqBody = req._body != null ? req._body : req.body !== undefined ? JSON.stringify(req.body) : "";
+    if (reqBody) lines.push("Sent body: " + truncate(reqBody, 800));
+  }
+
+  return lines.join("\n");
+}
+
+// Ask an LLM why a test failed. Pass options.provider (anything with complete())
+// to reuse a provider or to stub it in tests; otherwise one is created.
+export async function explainFailure(error, options = {}) {
+  const provider =
+    options.provider && typeof options.provider.complete === "function"
+      ? options.provider
+      : createProvider(options);
+
+  const reply = await provider.complete(buildPrompt(error, options), {
+    system: SYSTEM,
+    maxTokens: options.maxTokens || 512,
+  });
+  return reply.trim();
+}

src/ai/index.d.ts

@@ -7,3 +7,4 @@ export {
   type ChatMessage,
 } from "./provider.js";
 export { aiGenerateTests, type GenerateOptions } from "./generate.js";
+export { explainFailure, type ExplainOptions } from "./explain.js";

src/ai/index.js

@@ -2,3 +2,4 @@
 // no dependency. Bring your own API key.
 export { createProvider } from "./provider.js";
 export { aiGenerateTests } from "./generate.js";
+export { explainFailure } from "./explain.js";

src/index.js

@@ -36,6 +36,7 @@ import { fromPostman } from "./importers/postman.js";
 import { fromOpenapi } from "./importers/openapi.js";
 import { createProvider } from "./ai/provider.js";
 import { aiGenerateTests } from "./ai/generate.js";
+import { explainFailure } from "./ai/explain.js";
 
 // Namespace of all lodash-inspired utilities, available as both `_` and `utils`.
 import * as _ from "./utils/index.js";
@@ -91,6 +92,7 @@ export {
   fromOpenapi,
   createProvider,
   aiGenerateTests,
+  explainFailure,
 };
 
 // Also expose the utility belt under the `utils` name.

test/unit/ai-explain.test.mjs

@@ -0,0 +1,67 @@
+// Unit tests for AI failure explanation. The provider is stubbed, no network.
+import { test } from "node:test";
+import assert from "node:assert/strict";
+
+import { explainFailure } from "../../src/ai/explain.js";
+import { AssertionError } from "../../src/assertions.js";
+
+test("explainFailure builds a prompt from the error and response, returns the reply", async () => {
+  let seenPrompt = "";
+  let seenSystem = "";
+  const stub = {
+    complete: async (prompt, opts) => {
+      seenPrompt = prompt;
+      seenSystem = opts.system;
+      return "  The API returned 500 instead of 200, likely an unhandled error. Check the server logs.  ";
+    },
+  };
+
+  const error = new AssertionError("GET /users -> expected status 200 but got 500", {
+    expected: 200,
+    actual: 500,
+  });
+
+  const out = await explainFailure(error, {
+    provider: stub,
+    response: {
+      method: "GET",
+      url: "https://api.example.com/users",
+      status: 500,
+      statusText: "Internal Server Error",
+      time: 42,
+      headers: { "content-type": "text/html" },
+      text: "<html>boom</html>",
+    },
+  });
+
+  // trims the reply
+  assert.equal(out, "The API returned 500 instead of 200, likely an unhandled error. Check the server logs.");
+  // system prompt frames it as triage
+  assert.match(seenSystem, /triag/i);
+  // prompt carries the assertion, expected/actual, status and body
+  assert.match(seenPrompt, /expected status 200 but got 500/);
+  assert.match(seenPrompt, /Expected: 200/);
+  assert.match(seenPrompt, /Actual: 500/);
+  assert.match(seenPrompt, /Status: 500 Internal Server Error/);
+  assert.match(seenPrompt, /<html>boom<\/html>/);
+});
+
+test("explainFailure includes request context when provided", async () => {
+  let seenPrompt = "";
+  const stub = { complete: async (p) => { seenPrompt = p; return "ok"; } };
+
+  await explainFailure(new Error("boom"), {
+    provider: stub,
+    request: { _headers: { authorization: "Bearer x" }, _body: '{"a":1}' },
+  });
+
+  assert.match(seenPrompt, /Sent headers:/);
+  assert.match(seenPrompt, /Bearer x/);
+  assert.match(seenPrompt, /Sent body:/);
+});
+
+test("explainFailure works with a plain error and no context", async () => {
+  const stub = { complete: async () => "explanation" };
+  const out = await explainFailure(new Error("something broke"), { provider: stub });
+  assert.equal(out, "explanation");
+});