From 65e18524f9ff91df62e142867d33764596baba42 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Mon, 15 Jun 2026 11:18:01 +0200 Subject: [PATCH] fix(frontend): surface invocation errors instead of dumping HTML 404 pages When a playground or evaluator revision had no invocation URL, or the invoke endpoint returned an HTML 404 page instead of JSON, the run recorded the raw HTML as its output. An LLM-judge evaluator then scored a blob of HTML instead of reporting the error. The execution paths now return clear messages: "No invocation URL configured for this revision..." when the URL is empty, and "Service unreachable at (HTTP )" when the body is HTML. Consolidates the empty-URL message and the HTML detection into one shared helper in @agenta/entities/shared/execution/invocationErrors so all four fetch paths share the same logic. --- .../useWebWorker/assets/playground.worker.ts | 29 +++++++++- web/packages/agenta-entities/package.json | 1 + .../agenta-entities/src/runnable/utils.ts | 12 +++- .../src/shared/execution/index.ts | 7 +++ .../src/shared/execution/invocationErrors.ts | 36 ++++++++++++ .../tests/unit/invocationErrors.test.ts | 58 +++++++++++++++++++ .../src/state/execution/executionItems.ts | 34 +++++++---- .../src/state/execution/executionRunner.ts | 21 ++++++- 8 files changed, 180 insertions(+), 18 deletions(-) create mode 100644 web/packages/agenta-entities/src/shared/execution/invocationErrors.ts create mode 100644 web/packages/agenta-entities/tests/unit/invocationErrors.test.ts diff --git a/web/oss/src/components/Playground/hooks/useWebWorker/assets/playground.worker.ts b/web/oss/src/components/Playground/hooks/useWebWorker/assets/playground.worker.ts index 3d617f1cfe..42907a2cae 100644 --- a/web/oss/src/components/Playground/hooks/useWebWorker/assets/playground.worker.ts +++ b/web/oss/src/components/Playground/hooks/useWebWorker/assets/playground.worker.ts @@ -1,3 +1,9 @@ +import { + MISSING_INVOCATION_URL_ERROR, + describeUnreachableService, + isHtmlBody, +} from "@agenta/entities/shared/execution/invocationErrors" + interface RunVariantRowPayload { rowId: string entityId: string @@ -64,7 +70,7 @@ const asRecord = (value: unknown): Record | null => { return value as Record } -const parseErrorMessage = (status: number, data: unknown, fallbackText = ""): string => { +const parseErrorMessage = (status: number, data: unknown, fallbackText = "", url = ""): string => { if (status === 429) { const detailRec = asRecord(data) const detail = @@ -86,6 +92,9 @@ const parseErrorMessage = (status: number, data: unknown, fallbackText = ""): st if (typeof rec?.detail === "string" && rec.detail.trim().length > 0) { return rec.detail } + if (isHtmlBody(fallbackText)) { + return describeUnreachableService(url, status) + } if (typeof fallbackText === "string" && fallbackText.trim().length > 0) { return fallbackText } @@ -93,6 +102,17 @@ const parseErrorMessage = (status: number, data: unknown, fallbackText = ""): st } const executeRequest = async (payload: RunVariantRowPayload, controller: AbortController) => { + if (!payload.invocationUrl) { + return { + response: undefined, + error: MISSING_INVOCATION_URL_ERROR, + metadata: { + timestamp: new Date().toISOString(), + type: "configuration_error", + }, + } + } + try { const response = await fetch(payload.invocationUrl, { method: "POST", @@ -124,7 +144,12 @@ const executeRequest = async (payload: RunVariantRowPayload, controller: AbortCo if (!response.ok) { return { response: undefined, - error: parseErrorMessage(response.status, data, responseText), + error: parseErrorMessage( + response.status, + data, + responseText, + payload.invocationUrl, + ), metadata: { timestamp: new Date().toISOString(), statusCode: response.status, diff --git a/web/packages/agenta-entities/package.json b/web/packages/agenta-entities/package.json index 2d9b25ba74..a2170423ce 100644 --- a/web/packages/agenta-entities/package.json +++ b/web/packages/agenta-entities/package.json @@ -61,6 +61,7 @@ "./etl": "./src/etl/index.ts", "./shared/openapi": "./src/shared/openapi/index.ts", "./shared/execution": "./src/shared/execution/index.ts", + "./shared/execution/invocationErrors": "./src/shared/execution/invocationErrors.ts", "./shared/invalidation": "./src/shared/invalidation/index.ts" }, "dependencies": { diff --git a/web/packages/agenta-entities/src/runnable/utils.ts b/web/packages/agenta-entities/src/runnable/utils.ts index 9f0b2a4848..f705afef5d 100644 --- a/web/packages/agenta-entities/src/runnable/utils.ts +++ b/web/packages/agenta-entities/src/runnable/utils.ts @@ -9,6 +9,7 @@ import {projectIdAtom} from "@agenta/shared/state" import {getValueAtPath, generateId, parseMustache, walkMustache} from "@agenta/shared/utils" import {getDefaultStore} from "jotai/vanilla" +import {describeUnreachableService, isHtmlBody} from "../shared/execution/invocationErrors" import {parseEvaluatorKeyFromUri} from "../workflow/core" import {groupTemplateVariables} from "./portHelpers" @@ -1497,8 +1498,11 @@ export async function executeRunnable( errorMessage = errorData.detail } } catch { - // Response is not JSON, use raw text if available - if (errorText) { + // Response is not JSON. An HTML body means the service is + // unreachable or misconfigured; show that instead of the page. + if (isHtmlBody(errorText)) { + errorMessage = describeUnreachableService(data.invocationUrl, response.status) + } else if (errorText) { errorMessage = errorText } } @@ -1641,7 +1645,9 @@ async function executeEvaluator( errorMessage = errorData.detail } } catch { - if (errorText) { + if (isHtmlBody(errorText)) { + errorMessage = describeUnreachableService(url, response.status) + } else if (errorText) { errorMessage = errorText } } diff --git a/web/packages/agenta-entities/src/shared/execution/index.ts b/web/packages/agenta-entities/src/shared/execution/index.ts index 31c8a6cd96..eef8e1a6a7 100644 --- a/web/packages/agenta-entities/src/shared/execution/index.ts +++ b/web/packages/agenta-entities/src/shared/execution/index.ts @@ -54,6 +54,13 @@ export { extractInputValues, } from "./valueExtraction" +// Invocation error reporting +export { + MISSING_INVOCATION_URL_ERROR, + isHtmlBody, + describeUnreachableService, +} from "./invocationErrors" + // Request body builder export { transformToRequestBody, diff --git a/web/packages/agenta-entities/src/shared/execution/invocationErrors.ts b/web/packages/agenta-entities/src/shared/execution/invocationErrors.ts new file mode 100644 index 0000000000..a151bb0988 --- /dev/null +++ b/web/packages/agenta-entities/src/shared/execution/invocationErrors.ts @@ -0,0 +1,36 @@ +/** + * Shared helpers for reporting invocation failures. + * + * The playground and runnable execution paths all POST to a revision's + * invocation URL. Two failure modes used to leak through as confusing run + * output: + * 1. The revision has no invocation URL (empty `uri` and `url`), so the + * request resolved to the web app origin and returned the app's 404 page. + * 2. The service is unreachable or misconfigured, so the response body is an + * HTML error page instead of JSON. + * + * In both cases an LLM-judge evaluator received a blob of HTML as the "output" + * instead of a clear error. These helpers produce a readable message instead. + * + * They are intentionally dependency-free pure functions so the playground web + * worker can import this file without pulling in the rest of the package. + * + * @packageDocumentation + */ + +/** Message shown when a revision has no invocation URL to call. */ +export const MISSING_INVOCATION_URL_ERROR = + "No invocation URL configured for this revision (empty uri and url)" + +/** + * True when a response body looks like an HTML document rather than JSON. + * Matches a leading `` or `` tag (case-insensitive). + */ +export function isHtmlBody(body: string | null | undefined): boolean { + return typeof body === "string" && /^\s*<(!doctype|html)[\s>]/i.test(body) +} + +/** Message shown when the service returns an HTML page instead of JSON. */ +export function describeUnreachableService(url: string | null | undefined, status: number): string { + return `Service unreachable${url ? ` at ${url}` : ""} (HTTP ${status})` +} diff --git a/web/packages/agenta-entities/tests/unit/invocationErrors.test.ts b/web/packages/agenta-entities/tests/unit/invocationErrors.test.ts new file mode 100644 index 0000000000..2262841fce --- /dev/null +++ b/web/packages/agenta-entities/tests/unit/invocationErrors.test.ts @@ -0,0 +1,58 @@ +/** + * Unit tests for the shared invocation error helpers in + * `@agenta/entities/shared/execution/invocationErrors`. + * + * These pure helpers turn two confusing failure modes into clear messages: + * an HTML error body returned instead of JSON, and a missing invocation URL. + */ + +import {describe, expect, it} from "vitest" + +import { + MISSING_INVOCATION_URL_ERROR, + describeUnreachableService, + isHtmlBody, +} from "../../src/shared/execution/invocationErrors" + +describe("isHtmlBody", () => { + it("detects a doctype document", () => { + expect(isHtmlBody("404")).toBe(true) + }) + + it("detects a leading html tag, ignoring case and whitespace", () => { + expect(isHtmlBody('\n ')).toBe(true) + }) + + it("does not match JSON bodies", () => { + expect(isHtmlBody('{"detail": "not found"}')).toBe(false) + }) + + it("does not match text that merely contains a tag later", () => { + expect(isHtmlBody("error: is not allowed")).toBe(false) + }) + + it("handles null, undefined, and empty input", () => { + expect(isHtmlBody(null)).toBe(false) + expect(isHtmlBody(undefined)).toBe(false) + expect(isHtmlBody("")).toBe(false) + }) +}) + +describe("describeUnreachableService", () => { + it("includes the url when present", () => { + expect(describeUnreachableService("https://app/test", 404)).toBe( + "Service unreachable at https://app/test (HTTP 404)", + ) + }) + + it("omits the url when it is empty or missing", () => { + expect(describeUnreachableService("", 502)).toBe("Service unreachable (HTTP 502)") + expect(describeUnreachableService(null, 502)).toBe("Service unreachable (HTTP 502)") + }) +}) + +describe("MISSING_INVOCATION_URL_ERROR", () => { + it("is a stable, readable message", () => { + expect(MISSING_INVOCATION_URL_ERROR).toMatch(/no invocation url/i) + }) +}) diff --git a/web/packages/agenta-playground/src/state/execution/executionItems.ts b/web/packages/agenta-playground/src/state/execution/executionItems.ts index 5127d0fcdc..99b252f971 100644 --- a/web/packages/agenta-playground/src/state/execution/executionItems.ts +++ b/web/packages/agenta-playground/src/state/execution/executionItems.ts @@ -807,7 +807,11 @@ function resolveInvocationUrl( } } + // The legacy /test fallback needs a runtime prefix; without one the path + // resolves to the web app's origin and returns the 404 HTML page. const runtimePrefix = readString(requestPayload?.runtimePrefix) || "" + if (!runtimePrefix) return "" + const routePath = readString(requestPayload?.routePath) const path = constructPlaygroundTestPath(runtimePrefix, routePath) @@ -1135,19 +1139,25 @@ function buildExecutionItem( const rawPayloadRecord = asRecord(requestPayload) const isRawBody = !!rawPayloadRecord?.__rawBody - const invocationUrlWithQuery = appendQueryParams( - resolveInvocationUrl(params.invocationUrl, requestPayload, entityData), - { - // __rawBody payloads (workflow invoke) don't need application_id query param — - // the backend resolves the handler from the request body interface/URI. - application_id: isRawBody - ? undefined - : resolveApplicationId(requestPayload, entityData), - project_id: params.headers.Authorization - ? readString(params.projectId || undefined) - : undefined, - }, + const resolvedInvocationUrl = resolveInvocationUrl( + params.invocationUrl, + requestPayload, + entityData, ) + // Keep an empty URL empty so the execution path can report a clear error + // instead of POSTing to the web app origin and parsing its 404 HTML page. + const invocationUrlWithQuery = resolvedInvocationUrl + ? appendQueryParams(resolvedInvocationUrl, { + // __rawBody payloads (workflow invoke) don't need application_id query param — + // the backend resolves the handler from the request body interface/URI. + application_id: isRawBody + ? undefined + : resolveApplicationId(requestPayload, entityData), + project_id: params.headers.Authorization + ? readString(params.projectId || undefined) + : undefined, + }) + : "" const isAppWorkflow = !!rawPayloadRecord?.__appWorkflow const requestBody = isRawBody ? (() => { diff --git a/web/packages/agenta-playground/src/state/execution/executionRunner.ts b/web/packages/agenta-playground/src/state/execution/executionRunner.ts index 3ce4e04e62..1ce7f53bec 100644 --- a/web/packages/agenta-playground/src/state/execution/executionRunner.ts +++ b/web/packages/agenta-playground/src/state/execution/executionRunner.ts @@ -10,6 +10,11 @@ import { type EntitySelection, } from "@agenta/entities/runnable" import {isLocalDraftId} from "@agenta/entities/shared" +import { + MISSING_INVOCATION_URL_ERROR, + describeUnreachableService, + isHtmlBody, +} from "@agenta/entities/shared/execution/invocationErrors" import {workflowMolecule} from "@agenta/entities/workflow" import {generateId} from "@agenta/shared/utils" import type {Getter, Setter} from "jotai" @@ -906,6 +911,16 @@ async function executeViaFetch(params: { const executionId = generateId() const startedAt = new Date().toISOString() + if (!invocationUrl) { + return { + executionId, + status: "error", + startedAt, + completedAt: new Date().toISOString(), + error: {message: MISSING_INVOCATION_URL_ERROR}, + } + } + try { const response = await fetch(invocationUrl, { method: "POST", @@ -933,7 +948,11 @@ async function executeViaFetch(params: { errorMessage = errorData.detail } } catch { - if (errorText) errorMessage = errorText + if (isHtmlBody(errorText)) { + errorMessage = describeUnreachableService(invocationUrl, response.status) + } else if (errorText) { + errorMessage = errorText + } } return {