Skip to content

Commit 1d0bea8

Browse files
committed
Pin the telemetry contract end-to-end instead of unit-level
Replace the recording-tracer unit tests with one e2e scenario that drives the whole production topology — HTTP API → execution engine → sandbox → OpenAPI invoke → a real upstream returning 502 → OTLP export — and asserts the spans as the suite's motel store actually received them: outcome/error_code/error_status + tenant on executor.tool.execute, http.status_code on plugin.openapi.invoke, and ok/fail distinguishable. The unit tests verified span objects in process; the prod bug class this guards against (attribute stamped on a span the exporter never carries, failure riding the success channel) only shows up at the exported layer. Adds a Telemetry e2e service (motel span search with arrival polling), provided when the suite's motel boots (E2E_MOTEL_URL).
1 parent c431459 commit 1d0bea8

9 files changed

Lines changed: 293 additions & 365 deletions

File tree

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
// Cloud: the telemetry contract, end to end. A tool call that hits an
2+
// upstream error wall must be visible in the EXPORTED spans — not just
3+
// handled gracefully for the caller. This is the regression class where the
4+
// product silently goes dark to operators: `ToolResult.fail` rides the
5+
// Effect success channel (a healthy-looking span), and an attribute stamped
6+
// on the wrong span simply never arrives in the trace store, which looks
7+
// identical to health. So the assertion runs against the OTLP store the dev
8+
// stack actually exported to (the suite's motel — the same exporter layer
9+
// that ships prod spans to Axiom), driving the whole production topology:
10+
// HTTP API → execution engine → sandbox → OpenAPI invoke → a real upstream
11+
// returning 502 → span batch → OTLP export.
12+
//
13+
// Pins two regressions found live in prod (2026-06-12): http.status_code was
14+
// annotated inside the inner `OpenApi.invoke` span so the `plugin.openapi.
15+
// invoke` span queries target carried it on 0 of ~19.5k spans; and failed
16+
// tool calls were indistinguishable from successes on `executor.tool.execute`.
17+
import { randomBytes } from "node:crypto";
18+
import { createServer } from "node:http";
19+
20+
import { expect } from "@effect/vitest";
21+
import { Effect } from "effect";
22+
import { composePluginApi } from "@executor-js/api/server";
23+
import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api";
24+
import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared";
25+
26+
import { scenario } from "../src/scenario";
27+
import { Api, Target, Telemetry } from "../src/services";
28+
29+
const api = composePluginApi([openApiHttpPlugin()] as const);
30+
31+
/** Two operations: /ok answers 200, /fail answers 502 — the success and
32+
* expected-upstream-failure outcome classes the telemetry must separate. */
33+
const upstreamSpec = (baseUrl: string): string =>
34+
JSON.stringify({
35+
openapi: "3.0.3",
36+
info: { title: "Telemetry Upstream", version: "1.0.0" },
37+
servers: [{ url: baseUrl }],
38+
paths: {
39+
"/ok": {
40+
get: {
41+
operationId: "ok",
42+
summary: "Succeeds",
43+
tags: ["probe"],
44+
responses: { "200": { description: "" } },
45+
},
46+
},
47+
"/fail": {
48+
get: {
49+
operationId: "fail",
50+
summary: "Always 502",
51+
tags: ["probe"],
52+
responses: { "200": { description: "" } },
53+
},
54+
},
55+
},
56+
});
57+
58+
/** A real upstream on 127.0.0.1: /ok → 200 JSON, anything else → 502 JSON. */
59+
const serveUpstream = Effect.acquireRelease(
60+
Effect.callback<{ readonly baseUrl: string; readonly close: () => void }>((resume) => {
61+
const server = createServer((request, response) => {
62+
const ok = request.url?.startsWith("/ok") ?? false;
63+
response.writeHead(ok ? 200 : 502, { "content-type": "application/json" });
64+
response.end(ok ? '{"fine":true}' : '{"error":{"message":"bad gateway"}}');
65+
});
66+
server.listen(0, "127.0.0.1", () => {
67+
const address = server.address();
68+
const port = typeof address === "object" && address ? address.port : 0;
69+
resume(
70+
Effect.succeed({
71+
baseUrl: `http://127.0.0.1:${port}`,
72+
close: () => {
73+
server.close();
74+
server.closeAllConnections();
75+
},
76+
}),
77+
);
78+
});
79+
}),
80+
(server) => Effect.sync(server.close),
81+
);
82+
83+
scenario(
84+
"Telemetry · a failing tool call is visible in the exported spans",
85+
{ timeout: 180_000 },
86+
Effect.scoped(
87+
Effect.gen(function* () {
88+
const target = yield* Target;
89+
const { client: apiClient } = yield* Api;
90+
const telemetry = yield* Telemetry;
91+
const identity = yield* target.newIdentity();
92+
const client = yield* apiClient(api, identity);
93+
94+
const upstream = yield* serveUpstream;
95+
96+
// Identifier-safe slug: it becomes a property path in the sandbox code.
97+
const slug = IntegrationSlug.make(`telscn${randomBytes(4).toString("hex")}`);
98+
yield* client.openapi.addSpec({
99+
payload: {
100+
spec: { kind: "blob", value: upstreamSpec(upstream.baseUrl) },
101+
slug,
102+
baseUrl: upstream.baseUrl,
103+
authenticationTemplate: [
104+
{
105+
slug: "apiKey",
106+
type: "apiKey",
107+
headers: { Authorization: ["Bearer ", { type: "variable", name: "token" }] },
108+
},
109+
],
110+
},
111+
});
112+
yield* client.connections.create({
113+
payload: {
114+
owner: "org",
115+
name: ConnectionName.make("main"),
116+
integration: slug,
117+
template: AuthTemplateSlug.make("apiKey"),
118+
value: "telemetry-scenario-token",
119+
},
120+
});
121+
122+
const tools = yield* client.tools.list({ query: {} });
123+
const addressOf = (op: string) => {
124+
const tool = tools.find(
125+
(entry) =>
126+
String(entry.integration) === String(slug) && String(entry.address).endsWith(`.${op}`),
127+
);
128+
expect(tool, `the ${op} tool is in the catalog`).toBeDefined();
129+
return String(tool!.address);
130+
};
131+
const failAddress = addressOf("fail");
132+
const okAddress = addressOf("ok");
133+
134+
// Drive both outcome classes through the full production path. The
135+
// failing call still completes for the caller — that is exactly why
136+
// the exported span is the only place an operator can see it.
137+
for (const address of [okAddress, failAddress]) {
138+
const execution = yield* client.executions.execute({
139+
payload: { code: `return await ${address}({});` },
140+
});
141+
expect(execution.status, `the ${address} execution completes`).toBe("completed");
142+
}
143+
144+
// The failure: outcome attributes on the tool span...
145+
const failSpan = yield* telemetry.expectSpan({
146+
operation: "executor.tool.execute",
147+
attributes: { "mcp.tool.name": failAddress },
148+
});
149+
expect(failSpan.span.tags, "a failed tool call is marked on the exported span").toMatchObject(
150+
{
151+
"executor.tool.outcome": "fail",
152+
"executor.tool.error_code": "upstream_http_error",
153+
"executor.tool.error_status": "502",
154+
},
155+
);
156+
expect(
157+
failSpan.span.tags["executor.tenant"],
158+
"the span carries tenant attribution (no trace-join needed to ask 'whose error?')",
159+
).toBeTruthy();
160+
161+
// ...and the upstream status on the HTTP span queries actually target.
162+
const invokeSpan = yield* telemetry.expectSpan({
163+
operation: "plugin.openapi.invoke",
164+
attributes: { "plugin.openapi.base_url": upstream.baseUrl, "http.status_code": "502" },
165+
});
166+
expect(
167+
invokeSpan.span.tags["plugin.openapi.method"],
168+
"the invoke span names the method",
169+
).toBe("GET");
170+
171+
// The success is distinguishable from the failure.
172+
const okSpan = yield* telemetry.expectSpan({
173+
operation: "executor.tool.execute",
174+
attributes: { "mcp.tool.name": okAddress },
175+
});
176+
expect(okSpan.span.tags["executor.tool.outcome"], "a successful call is marked ok").toBe(
177+
"ok",
178+
);
179+
}),
180+
),
181+
);

e2e/setup/cloud.globalsetup.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,10 @@ export default async function setup(): Promise<(() => Promise<void>) | void> {
2828

2929
// Suite-owned trace store — every run captures distributed traces.
3030
const motel = await bootMotel();
31+
// Publish to the test workers (they inherit this process's env): scenarios
32+
// that assert on exported spans yield the Telemetry service, which exists
33+
// only when this is set. No motel → those scenarios skip, never fail.
34+
if (motel) process.env.E2E_MOTEL_URL = motel.url;
3135

3236
const publicUrl = `http://127.0.0.1:${ports.E2E_CLOUD_PORT!}`;
3337
let booted;

e2e/src/scenario.ts

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,20 @@ import { makeApiSurface } from "./surfaces/api";
2424
import { makeBrowserSurface } from "./surfaces/browser";
2525
import { makeCliSurface } from "./surfaces/cli";
2626
import { makeMcpSurface } from "./surfaces/mcp";
27+
import { makeTelemetrySurface } from "./surfaces/telemetry";
2728
import { completeOAuthConsent, hasOpenCode, makeOpenCodeHome, warmUp } from "./clients/opencode";
28-
import { Api, Billing, Browser, Cli, Mcp, OpenCode, RunDir, Target, TtlControl } from "./services";
29+
import {
30+
Api,
31+
Billing,
32+
Browser,
33+
Cli,
34+
Mcp,
35+
OpenCode,
36+
RunDir,
37+
Target,
38+
Telemetry,
39+
TtlControl,
40+
} from "./services";
2941
import { buildManifest } from "./viewer/manifest";
3042

3143
export const RUNS_DIR = fileURLToPath(new URL("../runs/", import.meta.url));
@@ -41,7 +53,17 @@ export interface ScenarioOptions {
4153
readonly timeout?: number;
4254
}
4355

44-
type AllServices = Target | RunDir | Cli | Api | Browser | Mcp | Billing | OpenCode | TtlControl;
56+
type AllServices =
57+
| Target
58+
| RunDir
59+
| Cli
60+
| Api
61+
| Browser
62+
| Mcp
63+
| Billing
64+
| OpenCode
65+
| TtlControl
66+
| Telemetry;
4567

4668
/**
4769
* What this target on this host can provide. Services beyond the base are
@@ -70,6 +92,9 @@ const contextFor = (target: TargetShape, dir: string): Context.Context<AllServic
7092
if (target.setAccessTokenTtl) {
7193
context = Context.add(context, TtlControl, target.setAccessTokenTtl);
7294
}
95+
if (process.env.E2E_MOTEL_URL) {
96+
context = Context.add(context, Telemetry, makeTelemetrySurface(process.env.E2E_MOTEL_URL));
97+
}
7398
return context;
7499
};
75100

e2e/src/services.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import type { ApiSurface } from "./surfaces/api";
1111
import type { BrowserSurface } from "./surfaces/browser";
1212
import type { CliSurface } from "./surfaces/cli";
1313
import type { McpSurface } from "./surfaces/mcp";
14+
import type { TelemetrySurface } from "./surfaces/telemetry";
1415
import type { completeOAuthConsent, makeOpenCodeHome, warmUp } from "./clients/opencode";
1516

1617
/** The target under test (always provided). */
@@ -34,6 +35,10 @@ export class Mcp extends Context.Service<Mcp, McpSurface>()("e2e/mcp-oauth") {}
3435
/** Marker: billing limits are enforced on this target. */
3536
export class Billing extends Context.Service<Billing, true>()("e2e/billing") {}
3637

38+
/** Query the suite's OTLP trace store for spans the target actually exported
39+
* (present when the suite booted motel — E2E_MOTEL_URL). */
40+
export class Telemetry extends Context.Service<Telemetry, TelemetrySurface>()("e2e/telemetry") {}
41+
3742
/** The real OpenCode binary, hermetically driveable (present when installed on this host). */
3843
export interface OpenCodeClient {
3944
readonly makeHome: typeof makeOpenCodeHome;

e2e/src/surfaces/telemetry.ts

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
// Telemetry surface: query the suite's motel OTLP store for the spans the
2+
// target ACTUALLY exported. This is how a scenario asserts the observability
3+
// contract end-to-end — not "did the code create a span object" but "did the
4+
// span leave the server, with the attributes production queries depend on".
5+
// The dangerous regression mode here is silence (an attribute stamped on the
6+
// wrong span, an error riding the success channel): absent data looks exactly
7+
// like health, so the contract has to be pinned where the data is read.
8+
import { Effect, Schedule } from "effect";
9+
10+
/** One exported span, as motel's /api/spans/search returns it. `tags` is the
11+
* span's attributes with every value stringified. */
12+
export interface ExportedSpan {
13+
readonly traceId: string;
14+
readonly rootOperationName: string;
15+
readonly span: {
16+
readonly spanId: string;
17+
readonly operationName: string;
18+
readonly status: "ok" | "error";
19+
readonly durationMs: number;
20+
readonly tags: Readonly<Record<string, string>>;
21+
};
22+
}
23+
24+
export interface SpanQuery {
25+
/** Exact-match attribute filters (motel `attr.<key>=<value>`). */
26+
readonly attributes?: Readonly<Record<string, string>>;
27+
readonly operation?: string;
28+
readonly traceId?: string;
29+
}
30+
31+
export interface TelemetrySurface {
32+
/** One-shot search against the trace store. */
33+
readonly searchSpans: (query: SpanQuery) => Effect.Effect<readonly ExportedSpan[], unknown>;
34+
/** Search until at least one span matches. Exporters batch (the app
35+
* flushes ~1s after the request), so arrival is eventually-consistent —
36+
* polling IS the contract: "the span reaches the store, soon". */
37+
readonly expectSpan: (query: SpanQuery) => Effect.Effect<ExportedSpan, unknown>;
38+
}
39+
40+
export const makeTelemetrySurface = (motelUrl: string): TelemetrySurface => {
41+
const searchSpans = (query: SpanQuery) =>
42+
Effect.gen(function* () {
43+
const params = new URLSearchParams({ lookback: "15m", limit: "100" });
44+
if (query.operation) params.set("operation", query.operation);
45+
if (query.traceId) params.set("traceId", query.traceId);
46+
for (const [key, value] of Object.entries(query.attributes ?? {})) {
47+
params.set(`attr.${key}`, value);
48+
}
49+
const response = yield* Effect.promise(() => fetch(`${motelUrl}/api/spans/search?${params}`));
50+
if (!response.ok) {
51+
return yield* Effect.fail(
52+
`motel span search responded ${response.status}: ${yield* Effect.promise(() => response.text())}`,
53+
);
54+
}
55+
const body = (yield* Effect.promise(() => response.json())) as {
56+
readonly data?: readonly ExportedSpan[];
57+
};
58+
return body.data ?? [];
59+
});
60+
61+
return {
62+
searchSpans,
63+
expectSpan: (query) =>
64+
searchSpans(query).pipe(
65+
Effect.filterOrFail(
66+
(spans) => spans.length > 0,
67+
() => `no exported span matched ${JSON.stringify(query)}`,
68+
),
69+
Effect.map((spans) => spans[0]!),
70+
// ~20s ceiling (40 × 500ms): BatchSpanProcessor flushes ~1s after the
71+
// request and the dev stack drains on waitUntil; slower is a real bug.
72+
Effect.retry(Schedule.both(Schedule.spaced("500 millis"), Schedule.recurs(40))),
73+
),
74+
};
75+
};

0 commit comments

Comments
 (0)