diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cfdbc6..3f0e6a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ All notable changes to this project are documented here. This project follows fetch (`createProvider`, bring your own key, OpenAI/Anthropic/custom baseURL) and `aiGenerateTests` to draft a suite from an endpoint or sample response. New CLI command `two-go ai gen [-o out]`. +- **AI failure explanation** (`explainFailure`): send a failed assertion plus + the request and response context to an LLM and get a likely cause and a + suggested fix. Advisory only, it never changes pass or fail. ## [0.4.0] diff --git a/README.md b/README.md index e99daed..ae09da2 100644 --- a/README.md +++ b/README.md @@ -630,6 +630,20 @@ like any other test. Treat it as a first draft and tighten the checks. The default models are `gpt-5.3` for OpenAI and `claude-opus-4-8` for Anthropic, and you can override either with `{ model }`. +When a test fails you can also ask the model what probably went wrong. This is +advisory, it runs after the failure and never changes pass or fail. + +```js +import { explainFailure } from "two-go/ai"; + +try { + await api.get("/users").expectStatus(200); +} catch (err) { + const why = await explainFailure(err, { response: err.response, provider: "openai" }); + console.log(why); // likely cause plus a suggested fix +} +``` + ## TypeScript Types are written by hand and shipped with the package, so you get diff --git a/src/ai/explain.d.ts b/src/ai/explain.d.ts new file mode 100644 index 0000000..e794a80 --- /dev/null +++ b/src/ai/explain.d.ts @@ -0,0 +1,12 @@ +// Type declarations for AI failure explanation. +import type { Provider, ProviderOptions } from "./provider.js"; + +export interface ExplainOptions extends Omit { + response?: unknown; + request?: unknown; + maxTokens?: number; + provider?: Provider | ProviderOptions["provider"]; +} + +/** Ask an LLM for the likely cause of a failed assertion and a suggested fix. */ +export declare function explainFailure(error: unknown, options?: ExplainOptions): Promise; diff --git a/src/ai/explain.js b/src/ai/explain.js new file mode 100644 index 0000000..7d80f92 --- /dev/null +++ b/src/ai/explain.js @@ -0,0 +1,72 @@ +// Explain a failed assertion with an LLM. This is advisory only: it runs after +// a test already failed and never changes pass or fail. It takes the error plus +// whatever request and response context you have and asks for a short, specific +// likely cause and a concrete fix. + +import { createProvider } from "./provider.js"; + +const SYSTEM = [ + "You are a senior API engineer triaging a failed test.", + "Given the failed assertion and the request and response, explain the most", + "likely cause in two to four sentences, then suggest one concrete fix.", + "Be specific. Do not invent details that are not present in the input.", +].join("\n"); + +function truncate(value, max = 2000) { + const s = value == null ? "" : String(value); + return s.length > max ? s.slice(0, max) + "... (truncated)" : s; +} + +function buildPrompt(error, options) { + const lines = []; + lines.push("Failed assertion:"); + lines.push(error && error.message ? error.message : String(error)); + if (error && error.expected !== undefined) { + lines.push("Expected: " + truncate(JSON.stringify(error.expected), 500)); + } + if (error && error.actual !== undefined) { + lines.push("Actual: " + truncate(JSON.stringify(error.actual), 500)); + } + + const res = options.response; + if (res) { + lines.push(""); + lines.push("Response:"); + const line = `${res.method || ""} ${res.url || ""}`.trim(); + if (line) lines.push(line); + if (res.status != null) { + lines.push("Status: " + res.status + (res.statusText ? " " + res.statusText : "")); + } + if (res.time != null) lines.push("Time: " + res.time + "ms"); + if (res.headers) lines.push("Headers: " + truncate(JSON.stringify(res.headers), 800)); + const body = res.text != null ? res.text : res.body !== undefined ? JSON.stringify(res.body) : ""; + if (body) lines.push("Body: " + truncate(body)); + } + + const req = options.request; + if (req) { + lines.push(""); + lines.push("Request:"); + const headers = req._headers || req.headers; + if (headers) lines.push("Sent headers: " + truncate(JSON.stringify(headers), 500)); + const reqBody = req._body != null ? req._body : req.body !== undefined ? JSON.stringify(req.body) : ""; + if (reqBody) lines.push("Sent body: " + truncate(reqBody, 800)); + } + + return lines.join("\n"); +} + +// Ask an LLM why a test failed. Pass options.provider (anything with complete()) +// to reuse a provider or to stub it in tests; otherwise one is created. +export async function explainFailure(error, options = {}) { + const provider = + options.provider && typeof options.provider.complete === "function" + ? options.provider + : createProvider(options); + + const reply = await provider.complete(buildPrompt(error, options), { + system: SYSTEM, + maxTokens: options.maxTokens || 512, + }); + return reply.trim(); +} diff --git a/src/ai/index.d.ts b/src/ai/index.d.ts index adbc4d4..a4739bd 100644 --- a/src/ai/index.d.ts +++ b/src/ai/index.d.ts @@ -7,3 +7,4 @@ export { type ChatMessage, } from "./provider.js"; export { aiGenerateTests, type GenerateOptions } from "./generate.js"; +export { explainFailure, type ExplainOptions } from "./explain.js"; diff --git a/src/ai/index.js b/src/ai/index.js index 8a39961..6577b2c 100644 --- a/src/ai/index.js +++ b/src/ai/index.js @@ -2,3 +2,4 @@ // no dependency. Bring your own API key. export { createProvider } from "./provider.js"; export { aiGenerateTests } from "./generate.js"; +export { explainFailure } from "./explain.js"; diff --git a/src/index.js b/src/index.js index f3024f5..84dc1f4 100644 --- a/src/index.js +++ b/src/index.js @@ -36,6 +36,7 @@ import { fromPostman } from "./importers/postman.js"; import { fromOpenapi } from "./importers/openapi.js"; import { createProvider } from "./ai/provider.js"; import { aiGenerateTests } from "./ai/generate.js"; +import { explainFailure } from "./ai/explain.js"; // Namespace of all lodash-inspired utilities, available as both `_` and `utils`. import * as _ from "./utils/index.js"; @@ -91,6 +92,7 @@ export { fromOpenapi, createProvider, aiGenerateTests, + explainFailure, }; // Also expose the utility belt under the `utils` name. diff --git a/test/unit/ai-explain.test.mjs b/test/unit/ai-explain.test.mjs new file mode 100644 index 0000000..e7b5a36 --- /dev/null +++ b/test/unit/ai-explain.test.mjs @@ -0,0 +1,67 @@ +// Unit tests for AI failure explanation. The provider is stubbed, no network. +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { explainFailure } from "../../src/ai/explain.js"; +import { AssertionError } from "../../src/assertions.js"; + +test("explainFailure builds a prompt from the error and response, returns the reply", async () => { + let seenPrompt = ""; + let seenSystem = ""; + const stub = { + complete: async (prompt, opts) => { + seenPrompt = prompt; + seenSystem = opts.system; + return " The API returned 500 instead of 200, likely an unhandled error. Check the server logs. "; + }, + }; + + const error = new AssertionError("GET /users -> expected status 200 but got 500", { + expected: 200, + actual: 500, + }); + + const out = await explainFailure(error, { + provider: stub, + response: { + method: "GET", + url: "https://api.example.com/users", + status: 500, + statusText: "Internal Server Error", + time: 42, + headers: { "content-type": "text/html" }, + text: "boom", + }, + }); + + // trims the reply + assert.equal(out, "The API returned 500 instead of 200, likely an unhandled error. Check the server logs."); + // system prompt frames it as triage + assert.match(seenSystem, /triag/i); + // prompt carries the assertion, expected/actual, status and body + assert.match(seenPrompt, /expected status 200 but got 500/); + assert.match(seenPrompt, /Expected: 200/); + assert.match(seenPrompt, /Actual: 500/); + assert.match(seenPrompt, /Status: 500 Internal Server Error/); + assert.match(seenPrompt, /boom<\/html>/); +}); + +test("explainFailure includes request context when provided", async () => { + let seenPrompt = ""; + const stub = { complete: async (p) => { seenPrompt = p; return "ok"; } }; + + await explainFailure(new Error("boom"), { + provider: stub, + request: { _headers: { authorization: "Bearer x" }, _body: '{"a":1}' }, + }); + + assert.match(seenPrompt, /Sent headers:/); + assert.match(seenPrompt, /Bearer x/); + assert.match(seenPrompt, /Sent body:/); +}); + +test("explainFailure works with a plain error and no context", async () => { + const stub = { complete: async () => "explanation" }; + const out = await explainFailure(new Error("something broke"), { provider: stub }); + assert.equal(out, "explanation"); +});