fix(frontend): surface invocation errors instead of dumping HTML 404 pages

web/oss/src/components/Playground/hooks/useWebWorker/assets/playground.worker.ts

@@ -1,3 +1,9 @@
+import {
+    MISSING_INVOCATION_URL_ERROR,
+    describeUnreachableService,
+    isHtmlBody,
+} from "@agenta/entities/shared/execution/invocationErrors"
+
 interface RunVariantRowPayload {
     rowId: string
     entityId: string
@@ -64,7 +70,7 @@ const asRecord = (value: unknown): Record<string, unknown> | null => {
     return value as Record<string, unknown>
 }
 
-const parseErrorMessage = (status: number, data: unknown, fallbackText = ""): string => {
+const parseErrorMessage = (status: number, data: unknown, fallbackText = "", url = ""): string => {
     if (status === 429) {
         const detailRec = asRecord(data)
         const detail =
@@ -86,13 +92,27 @@ const parseErrorMessage = (status: number, data: unknown, fallbackText = ""): st
     if (typeof rec?.detail === "string" && rec.detail.trim().length > 0) {
         return rec.detail
     }
+    if (isHtmlBody(fallbackText)) {
+        return describeUnreachableService(url, status)
+    }
     if (typeof fallbackText === "string" && fallbackText.trim().length > 0) {
         return fallbackText
     }
     return `Request failed with status ${status}`
 }
 
 const executeRequest = async (payload: RunVariantRowPayload, controller: AbortController) => {
+    if (!payload.invocationUrl) {
+        return {
+            response: undefined,
+            error: MISSING_INVOCATION_URL_ERROR,
+            metadata: {
+                timestamp: new Date().toISOString(),
+                type: "configuration_error",
+            },
+        }
+    }
+
     try {
         const response = await fetch(payload.invocationUrl, {
             method: "POST",
@@ -124,7 +144,12 @@ const executeRequest = async (payload: RunVariantRowPayload, controller: AbortCo
         if (!response.ok) {
             return {
                 response: undefined,
-                error: parseErrorMessage(response.status, data, responseText),
+                error: parseErrorMessage(
+                    response.status,
+                    data,
+                    responseText,
+                    payload.invocationUrl,
+                ),
                 metadata: {
                     timestamp: new Date().toISOString(),
                     statusCode: response.status,

web/packages/agenta-entities/package.json

@@ -61,6 +61,7 @@
         "./etl": "./src/etl/index.ts",
         "./shared/openapi": "./src/shared/openapi/index.ts",
         "./shared/execution": "./src/shared/execution/index.ts",
+        "./shared/execution/invocationErrors": "./src/shared/execution/invocationErrors.ts",
         "./shared/invalidation": "./src/shared/invalidation/index.ts"
     },
     "dependencies": {

web/packages/agenta-entities/src/runnable/utils.ts

@@ -9,6 +9,7 @@ import {projectIdAtom} from "@agenta/shared/state"
 import {getValueAtPath, generateId, parseMustache, walkMustache} from "@agenta/shared/utils"
 import {getDefaultStore} from "jotai/vanilla"
 
+import {describeUnreachableService, isHtmlBody} from "../shared/execution/invocationErrors"
 import {parseEvaluatorKeyFromUri} from "../workflow/core"
 
 import {groupTemplateVariables} from "./portHelpers"
@@ -1497,8 +1498,11 @@ export async function executeRunnable(
                     errorMessage = errorData.detail
                 }
             } catch {
-                // Response is not JSON, use raw text if available
-                if (errorText) {
+                // Response is not JSON. An HTML body means the service is
+                // unreachable or misconfigured; show that instead of the page.
+                if (isHtmlBody(errorText)) {
+                    errorMessage = describeUnreachableService(data.invocationUrl, response.status)
+                } else if (errorText) {
                     errorMessage = errorText
                 }
             }
@@ -1641,7 +1645,9 @@ async function executeEvaluator(
                     errorMessage = errorData.detail
                 }
             } catch {
-                if (errorText) {
+                if (isHtmlBody(errorText)) {
+                    errorMessage = describeUnreachableService(url, response.status)
+                } else if (errorText) {
                     errorMessage = errorText
                 }
             }

web/packages/agenta-entities/src/shared/execution/index.ts

@@ -54,6 +54,13 @@ export {
     extractInputValues,
 } from "./valueExtraction"
 
+// Invocation error reporting
+export {
+    MISSING_INVOCATION_URL_ERROR,
+    isHtmlBody,
+    describeUnreachableService,
+} from "./invocationErrors"
+
 // Request body builder
 export {
     transformToRequestBody,

web/packages/agenta-entities/src/shared/execution/invocationErrors.ts

@@ -0,0 +1,36 @@
+/**
+ * Shared helpers for reporting invocation failures.
+ *
+ * The playground and runnable execution paths all POST to a revision's
+ * invocation URL. Two failure modes used to leak through as confusing run
+ * output:
+ *   1. The revision has no invocation URL (empty `uri` and `url`), so the
+ *      request resolved to the web app origin and returned the app's 404 page.
+ *   2. The service is unreachable or misconfigured, so the response body is an
+ *      HTML error page instead of JSON.
+ *
+ * In both cases an LLM-judge evaluator received a blob of HTML as the "output"
+ * instead of a clear error. These helpers produce a readable message instead.
+ *
+ * They are intentionally dependency-free pure functions so the playground web
+ * worker can import this file without pulling in the rest of the package.
+ *
+ * @packageDocumentation
+ */
+
+/** Message shown when a revision has no invocation URL to call. */
+export const MISSING_INVOCATION_URL_ERROR =
+    "No invocation URL configured for this revision (empty uri and url)"
+
+/**
+ * True when a response body looks like an HTML document rather than JSON.
+ * Matches a leading `<!doctype ...>` or `<html ...>` tag (case-insensitive).
+ */
+export function isHtmlBody(body: string | null | undefined): boolean {
+    return typeof body === "string" && /^\s*<(!doctype|html)[\s>]/i.test(body)
+}
+
+/** Message shown when the service returns an HTML page instead of JSON. */
+export function describeUnreachableService(url: string | null | undefined, status: number): string {
+    return `Service unreachable${url ? ` at ${url}` : ""} (HTTP ${status})`
+}

web/packages/agenta-entities/tests/unit/invocationErrors.test.ts

@@ -0,0 +1,58 @@
+/**
+ * Unit tests for the shared invocation error helpers in
+ * `@agenta/entities/shared/execution/invocationErrors`.
+ *
+ * These pure helpers turn two confusing failure modes into clear messages:
+ * an HTML error body returned instead of JSON, and a missing invocation URL.
+ */
+
+import {describe, expect, it} from "vitest"
+
+import {
+    MISSING_INVOCATION_URL_ERROR,
+    describeUnreachableService,
+    isHtmlBody,
+} from "../../src/shared/execution/invocationErrors"
+
+describe("isHtmlBody", () => {
+    it("detects a doctype document", () => {
+        expect(isHtmlBody("<!DOCTYPE html><html><body>404</body></html>")).toBe(true)
+    })
+
+    it("detects a leading html tag, ignoring case and whitespace", () => {
+        expect(isHtmlBody('\n  <HTML lang="en">')).toBe(true)
+    })
+
+    it("does not match JSON bodies", () => {
+        expect(isHtmlBody('{"detail": "not found"}')).toBe(false)
+    })
+
+    it("does not match text that merely contains a tag later", () => {
+        expect(isHtmlBody("error: <html> is not allowed")).toBe(false)
+    })
+
+    it("handles null, undefined, and empty input", () => {
+        expect(isHtmlBody(null)).toBe(false)
+        expect(isHtmlBody(undefined)).toBe(false)
+        expect(isHtmlBody("")).toBe(false)
+    })
+})
+
+describe("describeUnreachableService", () => {
+    it("includes the url when present", () => {
+        expect(describeUnreachableService("https://app/test", 404)).toBe(
+            "Service unreachable at https://app/test (HTTP 404)",
+        )
+    })
+
+    it("omits the url when it is empty or missing", () => {
+        expect(describeUnreachableService("", 502)).toBe("Service unreachable (HTTP 502)")
+        expect(describeUnreachableService(null, 502)).toBe("Service unreachable (HTTP 502)")
+    })
+})
+
+describe("MISSING_INVOCATION_URL_ERROR", () => {
+    it("is a stable, readable message", () => {
+        expect(MISSING_INVOCATION_URL_ERROR).toMatch(/no invocation url/i)
+    })
+})

web/packages/agenta-playground/src/state/execution/executionItems.ts

@@ -807,7 +807,11 @@ function resolveInvocationUrl(
         }
     }
 
+    // The legacy /test fallback needs a runtime prefix; without one the path
+    // resolves to the web app's origin and returns the 404 HTML page.
     const runtimePrefix = readString(requestPayload?.runtimePrefix) || ""
+    if (!runtimePrefix) return ""
+
     const routePath = readString(requestPayload?.routePath)
     const path = constructPlaygroundTestPath(runtimePrefix, routePath)
 
@@ -1135,19 +1139,25 @@ function buildExecutionItem(
     const rawPayloadRecord = asRecord(requestPayload)
     const isRawBody = !!rawPayloadRecord?.__rawBody
 
-    const invocationUrlWithQuery = appendQueryParams(
-        resolveInvocationUrl(params.invocationUrl, requestPayload, entityData),
-        {
-            // __rawBody payloads (workflow invoke) don't need application_id query param —
-            // the backend resolves the handler from the request body interface/URI.
-            application_id: isRawBody
-                ? undefined
-                : resolveApplicationId(requestPayload, entityData),
-            project_id: params.headers.Authorization
-                ? readString(params.projectId || undefined)
-                : undefined,
-        },
+    const resolvedInvocationUrl = resolveInvocationUrl(
+        params.invocationUrl,
+        requestPayload,
+        entityData,
     )
+    // Keep an empty URL empty so the execution path can report a clear error
+    // instead of POSTing to the web app origin and parsing its 404 HTML page.
+    const invocationUrlWithQuery = resolvedInvocationUrl
+        ? appendQueryParams(resolvedInvocationUrl, {
+              // __rawBody payloads (workflow invoke) don't need application_id query param —
+              // the backend resolves the handler from the request body interface/URI.
+              application_id: isRawBody
+                  ? undefined
+                  : resolveApplicationId(requestPayload, entityData),
+              project_id: params.headers.Authorization
+                  ? readString(params.projectId || undefined)
+                  : undefined,
+          })
+        : ""
     const isAppWorkflow = !!rawPayloadRecord?.__appWorkflow
     const requestBody = isRawBody
         ? (() => {

web/packages/agenta-playground/src/state/execution/executionRunner.ts

@@ -10,6 +10,11 @@ import {
     type EntitySelection,
 } from "@agenta/entities/runnable"
 import {isLocalDraftId} from "@agenta/entities/shared"
+import {
+    MISSING_INVOCATION_URL_ERROR,
+    describeUnreachableService,
+    isHtmlBody,
+} from "@agenta/entities/shared/execution/invocationErrors"
 import {workflowMolecule} from "@agenta/entities/workflow"
 import {generateId} from "@agenta/shared/utils"
 import type {Getter, Setter} from "jotai"
@@ -906,6 +911,16 @@ async function executeViaFetch(params: {
     const executionId = generateId()
     const startedAt = new Date().toISOString()
 
+    if (!invocationUrl) {
+        return {
+            executionId,
+            status: "error",
+            startedAt,
+            completedAt: new Date().toISOString(),
+            error: {message: MISSING_INVOCATION_URL_ERROR},
+        }
+    }
+
     try {
         const response = await fetch(invocationUrl, {
             method: "POST",
@@ -933,7 +948,11 @@ async function executeViaFetch(params: {
                     errorMessage = errorData.detail
                 }
             } catch {
-                if (errorText) errorMessage = errorText
+                if (isHtmlBody(errorText)) {
+                    errorMessage = describeUnreachableService(invocationUrl, response.status)
+                } else if (errorText) {
+                    errorMessage = errorText
+                }
             }
 
             return {