diff --git a/web/oss/src/components/Playground/hooks/useWebWorker/assets/playground.worker.ts b/web/oss/src/components/Playground/hooks/useWebWorker/assets/playground.worker.ts index 3d617f1cfe..42907a2cae 100644 --- a/web/oss/src/components/Playground/hooks/useWebWorker/assets/playground.worker.ts +++ b/web/oss/src/components/Playground/hooks/useWebWorker/assets/playground.worker.ts @@ -1,3 +1,9 @@ +import { + MISSING_INVOCATION_URL_ERROR, + describeUnreachableService, + isHtmlBody, +} from "@agenta/entities/shared/execution/invocationErrors" + interface RunVariantRowPayload { rowId: string entityId: string @@ -64,7 +70,7 @@ const asRecord = (value: unknown): Record | null => { return value as Record } -const parseErrorMessage = (status: number, data: unknown, fallbackText = ""): string => { +const parseErrorMessage = (status: number, data: unknown, fallbackText = "", url = ""): string => { if (status === 429) { const detailRec = asRecord(data) const detail = @@ -86,6 +92,9 @@ const parseErrorMessage = (status: number, data: unknown, fallbackText = ""): st if (typeof rec?.detail === "string" && rec.detail.trim().length > 0) { return rec.detail } + if (isHtmlBody(fallbackText)) { + return describeUnreachableService(url, status) + } if (typeof fallbackText === "string" && fallbackText.trim().length > 0) { return fallbackText } @@ -93,6 +102,17 @@ const parseErrorMessage = (status: number, data: unknown, fallbackText = ""): st } const executeRequest = async (payload: RunVariantRowPayload, controller: AbortController) => { + if (!payload.invocationUrl) { + return { + response: undefined, + error: MISSING_INVOCATION_URL_ERROR, + metadata: { + timestamp: new Date().toISOString(), + type: "configuration_error", + }, + } + } + try { const response = await fetch(payload.invocationUrl, { method: "POST", @@ -124,7 +144,12 @@ const executeRequest = async (payload: RunVariantRowPayload, controller: AbortCo if (!response.ok) { return { response: undefined, - error: parseErrorMessage(response.status, data, responseText), + error: parseErrorMessage( + response.status, + data, + responseText, + payload.invocationUrl, + ), metadata: { timestamp: new Date().toISOString(), statusCode: response.status, diff --git a/web/packages/agenta-entities/package.json b/web/packages/agenta-entities/package.json index 2d9b25ba74..a2170423ce 100644 --- a/web/packages/agenta-entities/package.json +++ b/web/packages/agenta-entities/package.json @@ -61,6 +61,7 @@ "./etl": "./src/etl/index.ts", "./shared/openapi": "./src/shared/openapi/index.ts", "./shared/execution": "./src/shared/execution/index.ts", + "./shared/execution/invocationErrors": "./src/shared/execution/invocationErrors.ts", "./shared/invalidation": "./src/shared/invalidation/index.ts" }, "dependencies": { diff --git a/web/packages/agenta-entities/src/runnable/utils.ts b/web/packages/agenta-entities/src/runnable/utils.ts index 9f0b2a4848..f705afef5d 100644 --- a/web/packages/agenta-entities/src/runnable/utils.ts +++ b/web/packages/agenta-entities/src/runnable/utils.ts @@ -9,6 +9,7 @@ import {projectIdAtom} from "@agenta/shared/state" import {getValueAtPath, generateId, parseMustache, walkMustache} from "@agenta/shared/utils" import {getDefaultStore} from "jotai/vanilla" +import {describeUnreachableService, isHtmlBody} from "../shared/execution/invocationErrors" import {parseEvaluatorKeyFromUri} from "../workflow/core" import {groupTemplateVariables} from "./portHelpers" @@ -1497,8 +1498,11 @@ export async function executeRunnable( errorMessage = errorData.detail } } catch { - // Response is not JSON, use raw text if available - if (errorText) { + // Response is not JSON. An HTML body means the service is + // unreachable or misconfigured; show that instead of the page. + if (isHtmlBody(errorText)) { + errorMessage = describeUnreachableService(data.invocationUrl, response.status) + } else if (errorText) { errorMessage = errorText } } @@ -1641,7 +1645,9 @@ async function executeEvaluator( errorMessage = errorData.detail } } catch { - if (errorText) { + if (isHtmlBody(errorText)) { + errorMessage = describeUnreachableService(url, response.status) + } else if (errorText) { errorMessage = errorText } } diff --git a/web/packages/agenta-entities/src/shared/execution/index.ts b/web/packages/agenta-entities/src/shared/execution/index.ts index 31c8a6cd96..eef8e1a6a7 100644 --- a/web/packages/agenta-entities/src/shared/execution/index.ts +++ b/web/packages/agenta-entities/src/shared/execution/index.ts @@ -54,6 +54,13 @@ export { extractInputValues, } from "./valueExtraction" +// Invocation error reporting +export { + MISSING_INVOCATION_URL_ERROR, + isHtmlBody, + describeUnreachableService, +} from "./invocationErrors" + // Request body builder export { transformToRequestBody, diff --git a/web/packages/agenta-entities/src/shared/execution/invocationErrors.ts b/web/packages/agenta-entities/src/shared/execution/invocationErrors.ts new file mode 100644 index 0000000000..a151bb0988 --- /dev/null +++ b/web/packages/agenta-entities/src/shared/execution/invocationErrors.ts @@ -0,0 +1,36 @@ +/** + * Shared helpers for reporting invocation failures. + * + * The playground and runnable execution paths all POST to a revision's + * invocation URL. Two failure modes used to leak through as confusing run + * output: + * 1. The revision has no invocation URL (empty `uri` and `url`), so the + * request resolved to the web app origin and returned the app's 404 page. + * 2. The service is unreachable or misconfigured, so the response body is an + * HTML error page instead of JSON. + * + * In both cases an LLM-judge evaluator received a blob of HTML as the "output" + * instead of a clear error. These helpers produce a readable message instead. + * + * They are intentionally dependency-free pure functions so the playground web + * worker can import this file without pulling in the rest of the package. + * + * @packageDocumentation + */ + +/** Message shown when a revision has no invocation URL to call. */ +export const MISSING_INVOCATION_URL_ERROR = + "No invocation URL configured for this revision (empty uri and url)" + +/** + * True when a response body looks like an HTML document rather than JSON. + * Matches a leading `` or `` tag (case-insensitive). + */ +export function isHtmlBody(body: string | null | undefined): boolean { + return typeof body === "string" && /^\s*<(!doctype|html)[\s>]/i.test(body) +} + +/** Message shown when the service returns an HTML page instead of JSON. */ +export function describeUnreachableService(url: string | null | undefined, status: number): string { + return `Service unreachable${url ? ` at ${url}` : ""} (HTTP ${status})` +} diff --git a/web/packages/agenta-entities/tests/unit/invocationErrors.test.ts b/web/packages/agenta-entities/tests/unit/invocationErrors.test.ts new file mode 100644 index 0000000000..2262841fce --- /dev/null +++ b/web/packages/agenta-entities/tests/unit/invocationErrors.test.ts @@ -0,0 +1,58 @@ +/** + * Unit tests for the shared invocation error helpers in + * `@agenta/entities/shared/execution/invocationErrors`. + * + * These pure helpers turn two confusing failure modes into clear messages: + * an HTML error body returned instead of JSON, and a missing invocation URL. + */ + +import {describe, expect, it} from "vitest" + +import { + MISSING_INVOCATION_URL_ERROR, + describeUnreachableService, + isHtmlBody, +} from "../../src/shared/execution/invocationErrors" + +describe("isHtmlBody", () => { + it("detects a doctype document", () => { + expect(isHtmlBody("404")).toBe(true) + }) + + it("detects a leading html tag, ignoring case and whitespace", () => { + expect(isHtmlBody('\n ')).toBe(true) + }) + + it("does not match JSON bodies", () => { + expect(isHtmlBody('{"detail": "not found"}')).toBe(false) + }) + + it("does not match text that merely contains a tag later", () => { + expect(isHtmlBody("error: is not allowed")).toBe(false) + }) + + it("handles null, undefined, and empty input", () => { + expect(isHtmlBody(null)).toBe(false) + expect(isHtmlBody(undefined)).toBe(false) + expect(isHtmlBody("")).toBe(false) + }) +}) + +describe("describeUnreachableService", () => { + it("includes the url when present", () => { + expect(describeUnreachableService("https://app/test", 404)).toBe( + "Service unreachable at https://app/test (HTTP 404)", + ) + }) + + it("omits the url when it is empty or missing", () => { + expect(describeUnreachableService("", 502)).toBe("Service unreachable (HTTP 502)") + expect(describeUnreachableService(null, 502)).toBe("Service unreachable (HTTP 502)") + }) +}) + +describe("MISSING_INVOCATION_URL_ERROR", () => { + it("is a stable, readable message", () => { + expect(MISSING_INVOCATION_URL_ERROR).toMatch(/no invocation url/i) + }) +}) diff --git a/web/packages/agenta-playground/src/state/execution/executionItems.ts b/web/packages/agenta-playground/src/state/execution/executionItems.ts index 5127d0fcdc..99b252f971 100644 --- a/web/packages/agenta-playground/src/state/execution/executionItems.ts +++ b/web/packages/agenta-playground/src/state/execution/executionItems.ts @@ -807,7 +807,11 @@ function resolveInvocationUrl( } } + // The legacy /test fallback needs a runtime prefix; without one the path + // resolves to the web app's origin and returns the 404 HTML page. const runtimePrefix = readString(requestPayload?.runtimePrefix) || "" + if (!runtimePrefix) return "" + const routePath = readString(requestPayload?.routePath) const path = constructPlaygroundTestPath(runtimePrefix, routePath) @@ -1135,19 +1139,25 @@ function buildExecutionItem( const rawPayloadRecord = asRecord(requestPayload) const isRawBody = !!rawPayloadRecord?.__rawBody - const invocationUrlWithQuery = appendQueryParams( - resolveInvocationUrl(params.invocationUrl, requestPayload, entityData), - { - // __rawBody payloads (workflow invoke) don't need application_id query param — - // the backend resolves the handler from the request body interface/URI. - application_id: isRawBody - ? undefined - : resolveApplicationId(requestPayload, entityData), - project_id: params.headers.Authorization - ? readString(params.projectId || undefined) - : undefined, - }, + const resolvedInvocationUrl = resolveInvocationUrl( + params.invocationUrl, + requestPayload, + entityData, ) + // Keep an empty URL empty so the execution path can report a clear error + // instead of POSTing to the web app origin and parsing its 404 HTML page. + const invocationUrlWithQuery = resolvedInvocationUrl + ? appendQueryParams(resolvedInvocationUrl, { + // __rawBody payloads (workflow invoke) don't need application_id query param — + // the backend resolves the handler from the request body interface/URI. + application_id: isRawBody + ? undefined + : resolveApplicationId(requestPayload, entityData), + project_id: params.headers.Authorization + ? readString(params.projectId || undefined) + : undefined, + }) + : "" const isAppWorkflow = !!rawPayloadRecord?.__appWorkflow const requestBody = isRawBody ? (() => { diff --git a/web/packages/agenta-playground/src/state/execution/executionRunner.ts b/web/packages/agenta-playground/src/state/execution/executionRunner.ts index 3ce4e04e62..1ce7f53bec 100644 --- a/web/packages/agenta-playground/src/state/execution/executionRunner.ts +++ b/web/packages/agenta-playground/src/state/execution/executionRunner.ts @@ -10,6 +10,11 @@ import { type EntitySelection, } from "@agenta/entities/runnable" import {isLocalDraftId} from "@agenta/entities/shared" +import { + MISSING_INVOCATION_URL_ERROR, + describeUnreachableService, + isHtmlBody, +} from "@agenta/entities/shared/execution/invocationErrors" import {workflowMolecule} from "@agenta/entities/workflow" import {generateId} from "@agenta/shared/utils" import type {Getter, Setter} from "jotai" @@ -906,6 +911,16 @@ async function executeViaFetch(params: { const executionId = generateId() const startedAt = new Date().toISOString() + if (!invocationUrl) { + return { + executionId, + status: "error", + startedAt, + completedAt: new Date().toISOString(), + error: {message: MISSING_INVOCATION_URL_ERROR}, + } + } + try { const response = await fetch(invocationUrl, { method: "POST", @@ -933,7 +948,11 @@ async function executeViaFetch(params: { errorMessage = errorData.detail } } catch { - if (errorText) errorMessage = errorText + if (isHtmlBody(errorText)) { + errorMessage = describeUnreachableService(invocationUrl, response.status) + } else if (errorText) { + errorMessage = errorText + } } return {