From d83fa8a4ad4e735820ddf9e02906e0715c04ecfb Mon Sep 17 00:00:00 2001 From: Chad Hietala Date: Fri, 7 Aug 2026 15:19:31 -0400 Subject: [PATCH] feat(eve): split otel() from otelIntegration() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenTelemetry has a singleton half and a plural half. A process has one tracer provider, so it has one resource, one sampler, and one propagator set; it has as many destinations as the author cares to name. Folding both into a single `otel()` made the collision a runtime error eve had to detect. Splitting them makes it structurally impossible: one declaration per file, and only one file exports `otel()`. `otelIntegration({ traceExporter })` wraps the exporter in an eve-owned batching processor rather than taking `@opentelemetry/sdk-trace-base` as a runtime dependency — an exporter needs one thing done for it, and the dependency arrives with the whole SDK behind it. The two harness consumers move onto `#harness/otel-settings.js`, a store both layouts write, so neither the turn loop nor the channel-request wrapper has to know which one the build used. Signed-off-by: Chad Hietala --- .changeset/gentle-pandas-vanish.md | 13 + packages/eve/package.json | 6 + .../declarations/@opentelemetry/api.d.ts | 12 +- .../declarations/@vercel/otel.d.ts | 20 +- .../eve/src/harness/instrumentation-config.ts | 10 + .../src/harness/instrumentation-lifecycle.ts | 3 + ...mentation-providers-local.scenario.test.ts | 45 ++++ ...tion-providers-production.scenario.test.ts | 33 +++ .../harness/instrumentation-providers.test.ts | 93 ++++++++ .../src/harness/instrumentation-providers.ts | 74 ++++++ .../src/harness/instrumentation-runtime.ts | 1 + packages/eve/src/harness/otel-settings.ts | 37 +++ packages/eve/src/harness/tool-loop.test.ts | 42 +++- packages/eve/src/harness/tool-loop.ts | 37 +-- .../application/compiled-artifacts.ts | 15 +- ...instrumentation-layout.integration.test.ts | 7 +- .../src/internal/instrumentation-layout.ts | 10 +- .../create-application-nitro.scenario.test.ts | 11 +- .../nitro/host/create-application-nitro.ts | 8 +- .../host/local-tracing-runtime-plugin.ts | 8 +- .../routes/channel-request-instrumentation.ts | 9 +- .../eve/src/public/instrumentation/otel.ts | 59 +++++ .../src/tracing/agent-otel-provider.test.ts | 3 +- .../eve/src/tracing/agent-otel-provider.ts | 19 +- .../src/tracing/batch-span-processor.test.ts | 171 ++++++++++++++ .../eve/src/tracing/batch-span-processor.ts | 179 ++++++++++++++ .../eve/src/tracing/content-attributes.ts | 59 +++++ .../tracing/content-span-processor.test.ts | 35 +++ .../eve/src/tracing/content-span-processor.ts | 45 ++++ .../install-instrumentation-runtime.test.ts | 61 +++++ .../install-instrumentation-runtime.ts | 99 ++++++++ .../tracing/local-instrumentation-runtime.ts | 63 ++--- packages/eve/src/tracing/local-traces.test.ts | 25 +- packages/eve/src/tracing/local-traces.ts | 46 +++- .../eve/src/tracing/otel-declaration.test.ts | 140 +++++++---- packages/eve/src/tracing/otel-declaration.ts | 222 +++++++++++++++--- .../otel-registration.scenario.test.ts | 6 +- .../eve/src/tracing/otel-registration.test.ts | 30 ++- packages/eve/src/tracing/otel-registration.ts | 55 ++++- .../vercel-runtime-span-exporter.test.ts | 42 ++++ .../tracing/vercel-runtime-span-exporter.ts | 81 +++++++ ...piled-artifacts-bootstrap.scenario.test.ts | 54 ++++- pnpm-lock.yaml | 41 ++-- 43 files changed, 1783 insertions(+), 246 deletions(-) create mode 100644 .changeset/gentle-pandas-vanish.md create mode 100644 packages/eve/src/harness/instrumentation-providers-local.scenario.test.ts create mode 100644 packages/eve/src/harness/instrumentation-providers-production.scenario.test.ts create mode 100644 packages/eve/src/harness/otel-settings.ts create mode 100644 packages/eve/src/public/instrumentation/otel.ts create mode 100644 packages/eve/src/tracing/batch-span-processor.test.ts create mode 100644 packages/eve/src/tracing/batch-span-processor.ts create mode 100644 packages/eve/src/tracing/content-attributes.ts create mode 100644 packages/eve/src/tracing/content-span-processor.test.ts create mode 100644 packages/eve/src/tracing/content-span-processor.ts create mode 100644 packages/eve/src/tracing/install-instrumentation-runtime.test.ts create mode 100644 packages/eve/src/tracing/install-instrumentation-runtime.ts create mode 100644 packages/eve/src/tracing/vercel-runtime-span-exporter.test.ts create mode 100644 packages/eve/src/tracing/vercel-runtime-span-exporter.ts diff --git a/.changeset/gentle-pandas-vanish.md b/.changeset/gentle-pandas-vanish.md new file mode 100644 index 0000000000..f859c90fc1 --- /dev/null +++ b/.changeset/gentle-pandas-vanish.md @@ -0,0 +1,13 @@ +--- +"eve": patch +--- + +Split the OpenTelemetry authoring surface into `otel()` and `otelIntegration()`, +exported from the new `eve/instrumentation/otel` entrypoint. `otel()` declares +the settings a process can only hold one of — resource, sampler, propagators, +`functionId`, `traceChannelRequests` — and an integration declares one +destination, of which an agent may have as many as it has files. Passing +`traceExporter` to an integration wraps it in eve's batching span processor, so +a hosted backend is a one-liner. Local traces remain enabled by default in +development, and Agent Runs uses Vercel's runtime transport by default in +production. Reachable only with `experimental.instrumentationProviders` on. diff --git a/packages/eve/package.json b/packages/eve/package.json index f0ff6df8cb..d43dd55b53 100644 --- a/packages/eve/package.json +++ b/packages/eve/package.json @@ -206,6 +206,11 @@ "import": "./dist/src/public/instrumentation/index.js", "default": "./dist/src/public/instrumentation/index.js" }, + "./instrumentation/otel": { + "types": "./dist/src/public/instrumentation/otel.d.ts", + "import": "./dist/src/public/instrumentation/otel.js", + "default": "./dist/src/public/instrumentation/otel.js" + }, "./schedules": { "types": "./dist/src/public/schedules/index.d.ts", "import": "./dist/src/public/schedules/index.js", @@ -328,6 +333,7 @@ "@eve/catalog": "workspace:*", "@nuxt/kit": "^4.0.0", "@opentelemetry/context-async-hooks": "catalog:", + "@opentelemetry/core": "catalog:", "@opentelemetry/otlp-transformer": "0.214.0", "@opentelemetry/sdk-trace-base": "catalog:", "@photon-ai/chat-adapter-imessage": "3.2.0", diff --git a/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts b/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts index 2a3e425754..d7168a656b 100644 --- a/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts +++ b/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts @@ -30,7 +30,11 @@ export interface Tracer { ): Span; } -export interface Context {} +export interface Context { + setValue(key: symbol, value: unknown): Context; +} + +export declare function createContextKey(description: string): symbol; export declare const ROOT_CONTEXT: Context; @@ -40,6 +44,11 @@ export declare enum SpanStatusCode { ERROR = 2, } +export declare enum TraceFlags { + NONE = 0, + SAMPLED = 1, +} + export declare const context: { active(): Context; with(context: Context, fn: () => T): T; @@ -67,6 +76,7 @@ export declare const propagation: { export declare const trace: { getActiveSpan(): Span | undefined; getTracer(name: string, version?: string): Tracer; + getTracerProvider(): unknown; setSpan(context: Context, span: Span): Context; wrapSpanContext(spanContext: SpanContext): Span; }; diff --git a/packages/eve/scripts/vendor-compiled/declarations/@vercel/otel.d.ts b/packages/eve/scripts/vendor-compiled/declarations/@vercel/otel.d.ts index ac1c1d2845..d98c03d4c9 100644 --- a/packages/eve/scripts/vendor-compiled/declarations/@vercel/otel.d.ts +++ b/packages/eve/scripts/vendor-compiled/declarations/@vercel/otel.d.ts @@ -5,11 +5,29 @@ export interface SpanProcessor { shutdown(): Promise; } +export type SpanProcessorOrName = SpanProcessor | "auto"; + export interface IdGenerator { generateSpanId(): string; generateTraceId(): string; } +/** + * A `SpanExporter`. Structural for the same reason the propagator is: the + * instance comes from whichever `@opentelemetry/*` build the app installed, and + * eve only ever hands it spans and waits for the callback. + * + * `code` is `ExportResultCode`: `0` succeeded, `1` failed. + */ +export interface SpanExporter { + export( + spans: readonly unknown[], + resultCallback: (result: { code: number; error?: Error }) => void, + ): void; + forceFlush?(): Promise; + shutdown(): Promise; +} + /** * A `TextMapPropagator`, or one of the names `@vercel/otel` resolves for you. * Structural rather than imported: the instance comes from whichever @@ -40,7 +58,7 @@ export interface Configuration { readonly instrumentations?: readonly unknown[]; readonly propagators?: readonly PropagatorOrName[]; readonly serviceName?: string; - readonly spanProcessors?: readonly SpanProcessor[]; + readonly spanProcessors?: readonly SpanProcessorOrName[]; readonly traceSampler?: SamplerOrName; } diff --git a/packages/eve/src/harness/instrumentation-config.ts b/packages/eve/src/harness/instrumentation-config.ts index e61f2d89b3..52ba4c67b4 100644 --- a/packages/eve/src/harness/instrumentation-config.ts +++ b/packages/eve/src/harness/instrumentation-config.ts @@ -1,4 +1,5 @@ import { createInstrumentationSetupContext } from "#harness/instrumentation-setup-context.js"; +import { activateOtelSettings } from "#harness/otel-settings.js"; import type { InstrumentationDefinition } from "#public/instrumentation/index.js"; /** @@ -42,6 +43,15 @@ export async function registerInstrumentationConfig( input: { readonly agentName: string }, ): Promise { globalContainer[INSTRUMENTATION_CONFIG_GLOBAL_KEY] = config; + // The presence of a config is what turns telemetry on in this layout, so the + // settings the harness reads are activated with it rather than by a + // registered pipeline — this layout leaves `registerOTel` to `setup`. + activateOtelSettings({ + functionId: config.functionId, + recordInputs: config.recordInputs, + recordOutputs: config.recordOutputs, + traceChannelRequests: config.traceChannelRequests === true, + }); await config.setup?.(createInstrumentationSetupContext(input.agentName)); } diff --git a/packages/eve/src/harness/instrumentation-lifecycle.ts b/packages/eve/src/harness/instrumentation-lifecycle.ts index d7a104552b..78c501ac7b 100644 --- a/packages/eve/src/harness/instrumentation-lifecycle.ts +++ b/packages/eve/src/harness/instrumentation-lifecycle.ts @@ -256,6 +256,9 @@ export interface InstrumentationProviderDefinition { readonly "turn.failed"?: InstrumentationEventHandler; readonly "turn.started"?: InstrumentationEventHandler; }; + readonly flush?: () => void | PromiseLike; + readonly name?: string; + readonly shutdown?: () => void | PromiseLike; } /** Events that carry an operation `id`, pairing a start with its terminal. */ diff --git a/packages/eve/src/harness/instrumentation-providers-local.scenario.test.ts b/packages/eve/src/harness/instrumentation-providers-local.scenario.test.ts new file mode 100644 index 0000000000..358bc46154 --- /dev/null +++ b/packages/eve/src/harness/instrumentation-providers-local.scenario.test.ts @@ -0,0 +1,45 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + finalizeInstrumentationProviders, + getInstrumentationProviders, + registerInstrumentationProvider, + seedInstrumentationProviders, +} from "#harness/instrumentation-providers.js"; +import { DEVELOPMENT_WORKER_APP_ROOT_ENV } from "#internal/workflow/development-world-protocol.js"; +import { otelIntegration } from "#public/instrumentation/otel.js"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + vi.unstubAllEnvs(); + await Promise.all( + temporaryDirectories.splice(0).map((path) => rm(path, { force: true, recursive: true })), + ); +}); + +describe("instrumentation provider local default", () => { + it("registers default local traces and an authored destination in one pipeline", async () => { + const appRoot = await mkdtemp(join(tmpdir(), "eve-provider-local-")); + temporaryDirectories.push(appRoot); + vi.stubEnv(DEVELOPMENT_WORKER_APP_ROOT_ENV, appRoot); + vi.stubEnv("EVE_TRACES", "off"); + + seedInstrumentationProviders(); + await registerInstrumentationProvider({ + agentName: "weather", + slot: "backend", + value: otelIntegration(), + }); + + const runtime = finalizeInstrumentationProviders({ serviceName: "weather" }); + await runtime.forceFlush(); + await runtime.shutdown(); + + expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual(["local", "backend"]); + }); +}); diff --git a/packages/eve/src/harness/instrumentation-providers-production.scenario.test.ts b/packages/eve/src/harness/instrumentation-providers-production.scenario.test.ts new file mode 100644 index 0000000000..1afb442453 --- /dev/null +++ b/packages/eve/src/harness/instrumentation-providers-production.scenario.test.ts @@ -0,0 +1,33 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + finalizeInstrumentationProviders, + getInstrumentationProviders, + registerInstrumentationProvider, + seedInstrumentationProviders, +} from "#harness/instrumentation-providers.js"; +import { localTraces } from "#public/instrumentation/otel.js"; + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe("instrumentation provider production defaults", () => { + it("keeps authored local traces inert beside Agent Runs", async () => { + vi.stubEnv("EVE_DEV_WORKER_APP_ROOT", undefined); + vi.stubEnv("VERCEL_ENV", "production"); + + seedInstrumentationProviders(); + await registerInstrumentationProvider({ + agentName: "weather", + slot: "local", + value: localTraces(), + }); + + const runtime = finalizeInstrumentationProviders({ serviceName: "weather" }); + await runtime.forceFlush(); + await runtime.shutdown(); + + expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual(["agent-runs", "local"]); + }); +}); diff --git a/packages/eve/src/harness/instrumentation-providers.test.ts b/packages/eve/src/harness/instrumentation-providers.test.ts index 501c6b3801..893aa11248 100644 --- a/packages/eve/src/harness/instrumentation-providers.test.ts +++ b/packages/eve/src/harness/instrumentation-providers.test.ts @@ -1,16 +1,22 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { + finalizeInstrumentationProviders, getInstrumentationProviders, registerInstrumentationProvider, + seedInstrumentationProviders, + shutdownInstrumentationProviders, } from "#harness/instrumentation-providers.js"; +import { DEVELOPMENT_WORKER_APP_ROOT_ENV } from "#internal/workflow/development-world-protocol.js"; import { defineInstrumentation } from "#public/instrumentation/index.js"; +import { agentRuns, localTraces, otelIntegration } from "#public/instrumentation/otel.js"; import { disableInstrumentation, type ProviderSetupContext, } from "#public/instrumentation/provider.js"; const REGISTRY_GLOBAL_KEY = Symbol.for("eve.harness-instrumentation-providers"); +const RUNTIME_GLOBAL_KEY = Symbol.for("eve.instrumentation-runtime"); function register(slot: string, value: unknown): Promise { return registerInstrumentationProvider({ agentName: "weather-agent", slot, value }); @@ -20,6 +26,7 @@ describe("registerInstrumentationProvider", () => { beforeEach(() => { vi.unstubAllEnvs(); delete (globalThis as Record)[REGISTRY_GLOBAL_KEY]; + delete (globalThis as Record)[RUNTIME_GLOBAL_KEY]; }); it("registers a provider under its slot", async () => { @@ -93,3 +100,89 @@ describe("registerInstrumentationProvider", () => { ); }); }); + +describe("seedInstrumentationProviders", () => { + beforeEach(() => { + vi.unstubAllEnvs(); + delete (globalThis as Record)[REGISTRY_GLOBAL_KEY]; + delete (globalThis as Record)[RUNTIME_GLOBAL_KEY]; + vi.stubEnv("EVE_TRACES", "off"); + vi.stubEnv("VERCEL_ENV", undefined); + vi.stubEnv(DEVELOPMENT_WORKER_APP_ROOT_ENV, "/tmp/eve-seed-test"); + }); + + it("keeps default local traces beside an authored destination", async () => { + seedInstrumentationProviders(); + await register("backend", otelIntegration()); + + expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual(["local", "backend"]); + }); + + it("seeds Agent Runs only in Vercel production", () => { + vi.stubEnv(DEVELOPMENT_WORKER_APP_ROOT_ENV, undefined); + vi.stubEnv("VERCEL_ENV", "production"); + + seedInstrumentationProviders(); + + expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual(["agent-runs"]); + }); + + it("lets an authored reserved slot reconfigure or disable its default", async () => { + seedInstrumentationProviders(); + const authored = localTraces({ recordInputs: false }); + await register("local", authored); + expect(getInstrumentationProviders()).toEqual([{ provider: authored, slot: "local" }]); + + await register("local", disableInstrumentation()); + expect(getInstrumentationProviders()).toEqual([]); + }); + + it("lets an authored Agent Runs slot narrow the production default", async () => { + vi.stubEnv(DEVELOPMENT_WORKER_APP_ROOT_ENV, undefined); + vi.stubEnv("VERCEL_ENV", "production"); + seedInstrumentationProviders(); + const authored = agentRuns({ recordOutputs: false }); + + await register("agent-runs", authored); + + expect(getInstrumentationProviders()).toEqual([{ provider: authored, slot: "agent-runs" }]); + }); +}); + +describe("finalizeInstrumentationProviders", () => { + beforeEach(() => { + vi.unstubAllEnvs(); + delete (globalThis as Record)[REGISTRY_GLOBAL_KEY]; + delete (globalThis as Record)[RUNTIME_GLOBAL_KEY]; + }); + + it("installs a bus for authored providers without an OpenTelemetry destination", async () => { + const started = vi.fn(); + await register("rows", defineInstrumentation({ events: { "turn.started": started } })); + + const runtime = finalizeInstrumentationProviders({ serviceName: "weather-agent" }); + await runtime.hooks.publish({ + rootSessionId: "session-1", + sequence: 0, + sessionId: "session-1", + turnId: "turn-1", + type: "turn.started", + }); + + expect(started).toHaveBeenCalledOnce(); + }); + + it("drives authored flush and shutdown hooks", async () => { + const flush = vi.fn(); + const shutdown = vi.fn(); + await register("rows", defineInstrumentation({ flush, shutdown })); + + const runtime = finalizeInstrumentationProviders({ serviceName: "weather-agent" }); + await runtime.forceFlush(); + await shutdownInstrumentationProviders(); + await shutdownInstrumentationProviders(); + + expect(flush).toHaveBeenCalledOnce(); + expect(shutdown).toHaveBeenCalledOnce(); + }); +}); diff --git a/packages/eve/src/harness/instrumentation-providers.ts b/packages/eve/src/harness/instrumentation-providers.ts index 6e99fcd98a..82254ad43e 100644 --- a/packages/eve/src/harness/instrumentation-providers.ts +++ b/packages/eve/src/harness/instrumentation-providers.ts @@ -1,8 +1,23 @@ +import type { + InstrumentationEvent, + InstrumentationProviderDefinition, +} from "#harness/instrumentation-lifecycle.js"; +import { + getInstrumentationRuntime, + type InstrumentationRuntime, +} from "#harness/instrumentation-runtime.js"; import { createInstrumentationSetupContext } from "#harness/instrumentation-setup-context.js"; +import { resolveInstalledPackageInfo } from "#internal/application/package.js"; +import { DEVELOPMENT_WORKER_APP_ROOT_ENV } from "#internal/workflow/development-world-protocol.js"; +import { agentRuns, localTraces } from "#public/instrumentation/otel.js"; +import { installInstrumentationRuntime } from "#tracing/install-instrumentation-runtime.js"; +import { collectOtelPipeline } from "#tracing/otel-declaration.js"; import { isInstrumentationDisabled, isInstrumentationProvider, + type Handler, type InstrumentationProvider, + type ProviderContext, } from "#public/instrumentation/provider.js"; /** @@ -39,6 +54,15 @@ function providerRegistry(): Map { return created; } +/** Fills reserved slots before authored files may reconfigure or disable them. */ +export function seedInstrumentationProviders(): void { + const registry = providerRegistry(); + if (process.env.VERCEL_ENV === "production") registry.set("agent-runs", agentRuns()); + if (process.env[DEVELOPMENT_WORKER_APP_ROOT_ENV] !== undefined) { + registry.set("local", localTraces()); + } +} + /** * Registers one authored provider and awaits its `setup`. * @@ -74,3 +98,53 @@ export async function registerInstrumentationProvider(input: { export function getInstrumentationProviders(): readonly RegisteredInstrumentationProvider[] { return [...providerRegistry()].map(([slot, provider]) => ({ provider, slot })); } + +/** + * Builds the process's OpenTelemetry pipeline from the registered providers. + * + * Called once by the generated Nitro plugin after every slot has registered, + * which is also why it cannot happen inside `setup`: the pipeline is the union + * of every destination declared in the directory, so no single file knows + * enough to build it. A `setup` that reaches for a tracer therefore gets the + * no-op one; declare destinations as values and let this assemble them. + * + * A directory that declared no OpenTelemetry at all leaves the global tracer + * provider slot alone. + * + * @internal — not part of the public API. + */ +export function finalizeInstrumentationProviders(input: { + readonly serviceName: string; +}): InstrumentationRuntime { + const registered = getInstrumentationProviders(); + return installInstrumentationRuntime({ + collected: collectOtelPipeline(registered.map((entry) => entry.provider)), + frameworkVersion: resolveInstalledPackageInfo().version, + providers: registered.map(toProviderDefinition), + serviceName: input.serviceName, + }); +} + +/** Drains and releases the process runtime from Nitro's close hook. */ +export async function shutdownInstrumentationProviders(): Promise { + await getInstrumentationRuntime()?.shutdown(); +} + +const PROVIDER_CONTEXT: ProviderContext = Object.freeze({}); + +function toProviderDefinition( + entry: RegisteredInstrumentationProvider, +): InstrumentationProviderDefinition { + const events: Record void | PromiseLike> = {}; + for (const [type, handler] of Object.entries(entry.provider.events ?? {})) { + if (handler === undefined) continue; + const invoke = handler as Handler; + events[type] = (event: InstrumentationEvent) => invoke(event, PROVIDER_CONTEXT); + } + return { + events: events as InstrumentationProviderDefinition["events"], + flush: entry.provider.flush, + name: entry.slot, + shutdown: entry.provider.shutdown, + }; +} diff --git a/packages/eve/src/harness/instrumentation-runtime.ts b/packages/eve/src/harness/instrumentation-runtime.ts index 33c99efed9..1d2ffff39d 100644 --- a/packages/eve/src/harness/instrumentation-runtime.ts +++ b/packages/eve/src/harness/instrumentation-runtime.ts @@ -10,6 +10,7 @@ export interface InstrumentationRuntime { readonly forceFlush: () => Promise; readonly hooks: InstrumentationHooks; readonly runInContext: InstrumentationContextRunner; + readonly shutdown: () => Promise; } /** Instrumentation capabilities consumed inside one harness execution. */ diff --git a/packages/eve/src/harness/otel-settings.ts b/packages/eve/src/harness/otel-settings.ts new file mode 100644 index 0000000000..27531a07b3 --- /dev/null +++ b/packages/eve/src/harness/otel-settings.ts @@ -0,0 +1,37 @@ +import type { OtelHarnessSettings } from "#tracing/otel-declaration.js"; + +/** + * Process-global store for the telemetry settings the harness reads per turn. + * + * It exists because the two layouts declare the same settings in different + * shapes — `agent/instrumentation.ts` as fields on the config object, + * `agent/instrumentation/otel.ts` as `otel()` options — and the harness should + * not know which one the build used. Whichever registered writes here; the + * turn loop and the channel-request wrapper only read. + * + * Its presence is also the signal that telemetry is on at all. Rooted on + * `globalThis` for the reason the other instrumentation stores are: the + * generated Nitro plugin stays external by `file://` URL while the harness + * chunk is inlined, so the two are distinct ESM module instances. + */ +const OTEL_SETTINGS_GLOBAL_KEY = Symbol.for("eve.harness-otel-settings"); + +interface OtelSettingsGlobal { + [OTEL_SETTINGS_GLOBAL_KEY]?: OtelHarnessSettings; +} + +const globalContainer = globalThis as typeof globalThis & OtelSettingsGlobal; + +/** @internal — not part of the public API. */ +export function activateOtelSettings(settings: OtelHarnessSettings): void { + globalContainer[OTEL_SETTINGS_GLOBAL_KEY] = settings; +} + +/** + * The active settings, or `undefined` when nothing declared telemetry. + * + * @internal — not part of the public API. + */ +export function getOtelSettings(): OtelHarnessSettings | undefined { + return globalContainer[OTEL_SETTINGS_GLOBAL_KEY]; +} diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts index 3c69edff9c..cc7190c511 100644 --- a/packages/eve/src/harness/tool-loop.test.ts +++ b/packages/eve/src/harness/tool-loop.test.ts @@ -94,6 +94,24 @@ vi.mock("./instrumentation-config.js", () => ({ getInstrumentationConfig: (...args: unknown[]) => mockGetInstrumentationConfig(...args), })); +const mockGetOtelSettings = vi.fn().mockReturnValue(undefined); +vi.mock("./otel-settings.js", () => ({ + getOtelSettings: (...args: unknown[]) => mockGetOtelSettings(...args), +})); + +/** + * Registering an authored config writes both stores, so the tests toggle + * telemetry through one call rather than keeping two mocks in step by hand. + */ +function declareTelemetry(config: Readonly> | undefined): void { + mockGetInstrumentationConfig.mockReturnValue(config); + mockGetOtelSettings.mockReturnValue( + config === undefined + ? undefined + : { ...config, traceChannelRequests: config["traceChannelRequests"] === true }, + ); +} + vi.mock("./compaction.js", () => ({ compactMessages: vi.fn(), estimateTokens: vi.fn().mockReturnValue(5000), @@ -113,7 +131,7 @@ vi.mock("./compaction.js", () => ({ afterEach(() => { vi.clearAllMocks(); vi.unstubAllEnvs(); - mockGetInstrumentationConfig.mockReturnValue(undefined); + declareTelemetry(undefined); }); function createTestSession(overrides?: Partial): HarnessSession { @@ -4356,7 +4374,7 @@ describe("createToolLoopHarness", () => { "test.attempt": typeof input.modelInput.instructions === "string" ? "original" : "retry", }, })); - mockGetInstrumentationConfig.mockReturnValue({ + declareTelemetry({ events: { "step.started": resolveRuntimeContext, }, @@ -9416,11 +9434,11 @@ describe("createToolLoopHarness", () => { toolResults: [{ toolCallId: "call-1", toolName: "add", output: "42" }], }); - mockGetInstrumentationConfig.mockReturnValue({}); + declareTelemetry({}); const config = createTestConfig("conversation"); const runStep = createToolLoopHarness(config); const result = await runStep(createTestSession(), { message: "add stuff" }); - mockGetInstrumentationConfig.mockReturnValue(undefined); + declareTelemetry(undefined); expect(result.next).toBe(runStep); expect(result.session.state?.["eve.harness.turnTrace"]).toEqual({ @@ -9483,7 +9501,7 @@ describe("createToolLoopHarness", () => { toolResults: [{ toolCallId: "call-1", toolName: "add", output: "42" }], }); - mockGetInstrumentationConfig.mockReturnValue({}); + declareTelemetry({}); const step1Config = createTestConfig("conversation"); const step1 = createToolLoopHarness(step1Config); const result1 = await step1(createTestSession(), { message: "add stuff" }); @@ -9512,7 +9530,7 @@ describe("createToolLoopHarness", () => { const step2 = createToolLoopHarness(step2Config); // No input — continuation step const result2 = await step2(result1.session); - mockGetInstrumentationConfig.mockReturnValue(undefined); + declareTelemetry(undefined); expect(result2.next).toBeNull(); @@ -9541,11 +9559,11 @@ describe("createToolLoopHarness", () => { toolResults: [], }); - mockGetInstrumentationConfig.mockReturnValue({}); + declareTelemetry({}); const config = createTestConfig("conversation"); const runStep = createToolLoopHarness(config); await runStep(createTestSession(), { message: "hi" }); - mockGetInstrumentationConfig.mockReturnValue(undefined); + declareTelemetry(undefined); const agentCall = vi.mocked(ToolLoopAgent).mock.calls[0]?.[0] as { runtimeContext?: Record; @@ -9702,7 +9720,7 @@ describe("createToolLoopHarness", () => { toolCalls: [], toolResults: [], }); - mockGetInstrumentationConfig.mockReturnValue({ recordInputs: true, recordOutputs: false }); + declareTelemetry({ recordInputs: true, recordOutputs: false }); const hooks = createInstrumentationHooks([]); const runStep = createToolLoopHarness( createTestConfig("conversation", undefined, { @@ -9761,7 +9779,7 @@ describe("createToolLoopHarness", () => { }, }; }); - mockGetInstrumentationConfig.mockReturnValue({ + declareTelemetry({ events: { "step.started": resolveRuntimeContext, }, @@ -9816,7 +9834,7 @@ describe("createToolLoopHarness", () => { toolResults: [], }); - mockGetInstrumentationConfig.mockReturnValue({ + declareTelemetry({ events: { "step.started": () => { throw new Error("runtime context resolver failed"); @@ -9854,7 +9872,7 @@ describe("createToolLoopHarness", () => { "test.step": `${input.turn.id}:${input.step.index}`, }, })); - mockGetInstrumentationConfig.mockReturnValue({ + declareTelemetry({ events: { "step.started": resolveRuntimeContext, }, diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.ts index 8a8a2ec093..f6c77d8205 100644 --- a/packages/eve/src/harness/tool-loop.ts +++ b/packages/eve/src/harness/tool-loop.ts @@ -52,7 +52,6 @@ import { createSessionWaitingEvent, createStepStartedEvent, } from "#protocol/message.js"; -import type { InstrumentationDefinition } from "#public/instrumentation/index.js"; import { ASK_QUESTION_TOOL_NAME } from "#runtime/framework-tools/ask-question.js"; import { resolveAgentsAnnouncement } from "#harness/handles/prompt.js"; import { getAgentHandleStore } from "#harness/handles/store.js"; @@ -136,6 +135,8 @@ import { dropStaleSessionLimitContinuationResponses, } from "#harness/stale-input-responses.js"; import { getInstrumentationConfig } from "#harness/instrumentation-config.js"; +import { getOtelSettings } from "#harness/otel-settings.js"; +import type { OtelHarnessSettings } from "#tracing/otel-declaration.js"; import { normalizeUserContent, resolveAssistantStepText } from "#harness/messages.js"; import { normalizeProviderToolHistory } from "#harness/provider-tool-history.js"; import { @@ -263,12 +264,12 @@ const MODEL_CALL_MAX_ATTEMPTS = 3; const MODEL_CALL_RETRY_BASE_DELAY_MS = 500; function enrichTelemetry( - authored: InstrumentationDefinition | undefined, + settings: OtelHarnessSettings | undefined, agentName: string | undefined, runtimeContext?: Readonly>, bridgeIntegration?: Telemetry, ): TelemetryOptions | undefined { - if (authored === undefined && bridgeIntegration === undefined) { + if (settings === undefined && bridgeIntegration === undefined) { return undefined; } @@ -280,17 +281,17 @@ function enrichTelemetry( } return { - functionId: authored?.functionId ?? agentName, + functionId: settings?.functionId ?? agentName, includeRuntimeContext, integrations: bridgeIntegration === undefined ? undefined - : authored === undefined + : settings === undefined ? [bridgeIntegration] : [bridgeIntegration, createOtelIntegration()], isEnabled: true, - recordInputs: authored?.recordInputs ?? true, - recordOutputs: authored?.recordOutputs ?? true, + recordInputs: settings?.recordInputs ?? true, + recordOutputs: settings?.recordOutputs ?? true, }; } @@ -514,11 +515,15 @@ function buildHarnessToolsWithDynamicSubagents( export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn { const baseEmit = config.handleEvent; - const telemetryConfig = getInstrumentationConfig(); - if (telemetryConfig !== undefined) { + const otelSettings = getOtelSettings(); + // The custom-context enrichment below still reads the authored config object + // directly. Its replacement on the provider surface is unresolved, so a + // provider directory contributes no custom context yet. + const authoredConfig = getInstrumentationConfig(); + if (otelSettings !== undefined) { ensureOtelIntegration(); } - const tracer = telemetryConfig !== undefined ? trace.getTracer("eve") : undefined; + const tracer = otelSettings !== undefined ? trace.getTracer("eve") : undefined; const agentName = config.runtimeIdentity?.agentName; async function runStep( @@ -531,7 +536,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn { // restore the parent from session state via resolveStepOtelContext. let turnSpan: Span | undefined; if (tracer && hasStepInput(input)) { - const functionId = telemetryConfig?.functionId ?? agentName; + const functionId = otelSettings?.functionId ?? agentName; const attributes: Record = { "eve.version": eveVersion, "eve.environment": environment, @@ -628,7 +633,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn { resolveModel: config.resolveModel, runtimeIdentity: config.runtimeIdentity, session, - telemetry: enrichTelemetry(telemetryConfig, agentName) ?? undefined, + telemetry: enrichTelemetry(otelSettings, agentName) ?? undefined, }); session = { @@ -836,7 +841,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn { resolveModel: config.resolveModel, runtimeIdentity: config.runtimeIdentity, session, - telemetry: enrichTelemetry(telemetryConfig, agentName) ?? undefined, + telemetry: enrichTelemetry(otelSettings, agentName) ?? undefined, })); const approvedTools = getApprovedTools(session); @@ -908,7 +913,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn { instructions, telemetryRuntimeContext: buildTelemetryRuntimeContext({ eveVersion, - authored: telemetryConfig, + authored: authoredConfig, emissionState, environment, modelInput: { @@ -1030,7 +1035,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn { : { attemptId: `${session.sessionId}:${instrumentationTurnId}:${emissionState.stepIndex}:${opts.attemptIndex}`, attemptIndex: opts.attemptIndex, - functionId: telemetryConfig?.functionId ?? agentName, + functionId: otelSettings?.functionId ?? agentName, rootSessionId: parent?.rootSessionId ?? session.sessionId, sessionId: session.sessionId, stepIndex: emissionState.stepIndex, @@ -1083,7 +1088,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn { runtimeContext: telemetryRuntimeContext, stopWhen: isStepCount(1), telemetry: enrichTelemetry( - telemetryConfig, + otelSettings, agentName, telemetryRuntimeContext, bridgeIntegration, diff --git a/packages/eve/src/internal/application/compiled-artifacts.ts b/packages/eve/src/internal/application/compiled-artifacts.ts index c3d86e135f..ba835e0e19 100644 --- a/packages/eve/src/internal/application/compiled-artifacts.ts +++ b/packages/eve/src/internal/application/compiled-artifacts.ts @@ -423,7 +423,8 @@ export function createDevelopmentWorkflowWorldPluginSource(input: { * * The single-config layout registers one default export; the provider layout * registers one per file, in slot order, awaiting each `setup` so a provider - * cannot miss an event published while it is still starting up. + * cannot miss an event published while it is still starting up, then builds the + * one OpenTelemetry pipeline their declarations add up to. */ function createInstrumentationPluginSource(input: { agentName: string; @@ -462,7 +463,9 @@ function createInstrumentationPluginSource(input: { ([slot, modulePath], index) => `import * as provider${index} from ${stringifyEsmImportSpecifier(modulePath)}; // ${slot}`, ), - `import { registerInstrumentationProvider } from ${stringifyEsmImportSpecifier(registerProviderPath)};`, + `import { finalizeInstrumentationProviders, registerInstrumentationProvider, seedInstrumentationProviders, shutdownInstrumentationProviders } from ${stringifyEsmImportSpecifier(registerProviderPath)};`, + "", + "seedInstrumentationProviders();", "", ...slots.flatMap(([slot], index) => [ "await registerInstrumentationProvider({", @@ -472,9 +475,15 @@ function createInstrumentationPluginSource(input: { "});", ]), "", + `finalizeInstrumentationProviders({ serviceName: ${agentName} });`, + "", "// Default export satisfies the Nitro plugin contract so this file", "// can be used directly as a Nitro plugin without a separate wrapper.", - "export default function installInstrumentationPlugin() {}", + "export default function installInstrumentationPlugin(nitroApp) {", + " nitroApp?.hooks?.hook('close', async () => {", + " await shutdownInstrumentationProviders();", + " });", + "}", "", ].join("\n"); } diff --git a/packages/eve/src/internal/instrumentation-layout.integration.test.ts b/packages/eve/src/internal/instrumentation-layout.integration.test.ts index 1d4d7542bb..0833ec49eb 100644 --- a/packages/eve/src/internal/instrumentation-layout.integration.test.ts +++ b/packages/eve/src/internal/instrumentation-layout.integration.test.ts @@ -58,8 +58,11 @@ describe("resolveInstrumentationLayout with providers off", () => { }); describe("resolveInstrumentationLayout with providers on", () => { - it("returns nothing when the agent authored no instrumentation", () => { - expect(resolveInstrumentationLayout({ agentRoot, providersEnabled: true })).toBeUndefined(); + it("returns an empty provider layout for eve's built-in destinations", () => { + expect(resolveInstrumentationLayout({ agentRoot, providersEnabled: true })).toEqual({ + kind: "providers", + modulePathsBySlot: {}, + }); }); it("keys each file by the slot its name derives", () => { diff --git a/packages/eve/src/internal/instrumentation-layout.ts b/packages/eve/src/internal/instrumentation-layout.ts index 366fb46ec4..44546d0ef1 100644 --- a/packages/eve/src/internal/instrumentation-layout.ts +++ b/packages/eve/src/internal/instrumentation-layout.ts @@ -23,10 +23,10 @@ export type InstrumentationLayout = /** * Resolves the instrumentation layout for one agent root. * - * Returns `undefined` when the agent authored no instrumentation. Throws when - * the layout on disk is not the one the flag selects: the wrong layout would - * otherwise be skipped silently, and telemetry that quietly does nothing is the - * failure this whole surface exists to prevent. + * With providers on, an empty provider layout still installs eve's built-in + * destinations. Throws when the layout on disk is not the one the flag selects: + * the wrong layout would otherwise be skipped silently, and telemetry that + * quietly does nothing is the failure this whole surface exists to prevent. */ export function resolveInstrumentationLayout(input: { readonly agentRoot: string; @@ -53,7 +53,7 @@ export function resolveInstrumentationLayout(input: { } if (!hasDirectory) { - return undefined; + return { kind: "providers", modulePathsBySlot: {} }; } return { diff --git a/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts b/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts index 5c4f325248..5a12a75adb 100644 --- a/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts +++ b/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts @@ -228,14 +228,11 @@ describe("application Nitro creation", () => { ); }); - it("keeps local tracing beside provider slots unless local is authored", async () => { + it("lets the provider pipeline own default local tracing", async () => { const { createDevelopmentApplicationNitro } = await import("#internal/nitro/host/create-application-nitro.js"); - for (const [slots, expectsLocal] of [ - ["rows", true], - ["local", false], - ] as const) { + for (const slots of ["rows", "local"] as const) { const nitroStub = createNitroStub(); createNitroMock.mockResolvedValueOnce(nitroStub.nitro); const preparedHost = createPreparedHost(); @@ -248,8 +245,8 @@ describe("application Nitro creation", () => { await createDevelopmentApplicationNitro(preparedHost); const plugins = createNitroMock.mock.calls.at(-1)?.[0].plugins as string[]; - expect(plugins.some((plugin) => plugin.includes("local-tracing-runtime-plugin.ts"))).toBe( - expectsLocal, + expect(plugins).not.toEqual( + expect.arrayContaining([expect.stringContaining("local-tracing-runtime-plugin.ts")]), ); } }); diff --git a/packages/eve/src/internal/nitro/host/create-application-nitro.ts b/packages/eve/src/internal/nitro/host/create-application-nitro.ts index 82f70345bc..65483dee91 100644 --- a/packages/eve/src/internal/nitro/host/create-application-nitro.ts +++ b/packages/eve/src/internal/nitro/host/create-application-nitro.ts @@ -732,13 +732,7 @@ export async function createDevelopmentApplicationNitro( const nitroBuildDir = preparedHost.workspace.nitroBuildDir; const bundler = createApplicationNitroBundlerConfiguration(preparedHost, undefined); const plugins = createApplicationNitroPlugins(preparedHost); - const instrumentationLayout = preparedHost.compiledArtifacts.instrumentationLayout; - const providersKeepDefaultLocalTracing = - instrumentationLayout?.kind === "providers" && !instrumentationLayout.slots.includes("local"); - if ( - preparedHost.compiledArtifacts.instrumentationPluginPath === undefined || - providersKeepDefaultLocalTracing - ) { + if (preparedHost.compiledArtifacts.instrumentationPluginPath === undefined) { plugins.unshift( resolvePackageSourceFilePath("src/internal/nitro/host/local-tracing-runtime-plugin.ts"), ); diff --git a/packages/eve/src/internal/nitro/host/local-tracing-runtime-plugin.ts b/packages/eve/src/internal/nitro/host/local-tracing-runtime-plugin.ts index fe98337637..a175572139 100644 --- a/packages/eve/src/internal/nitro/host/local-tracing-runtime-plugin.ts +++ b/packages/eve/src/internal/nitro/host/local-tracing-runtime-plugin.ts @@ -9,10 +9,14 @@ if (appRoot === undefined) { throw new Error(`${DEVELOPMENT_WORKER_APP_ROOT_ENV} is required for local tracing.`); } -installLocalInstrumentationRuntime({ +const runtime = installLocalInstrumentationRuntime({ appRoot, frameworkVersion: resolveInstalledPackageInfo().version, serviceName: basename(appRoot), }); -export default function installLocalTracingRuntimePlugin(): void {} +export default function installLocalTracingRuntimePlugin(nitroApp?: { + readonly hooks?: { hook(name: "close", handler: () => Promise): unknown }; +}): void { + nitroApp?.hooks?.hook("close", () => runtime.shutdown()); +} diff --git a/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts b/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts index dbe8d34724..e0b5809585 100644 --- a/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts +++ b/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts @@ -8,7 +8,7 @@ import { type TextMapGetter, trace, } from "#compiled/@opentelemetry/api/index.js"; -import { getInstrumentationConfig } from "#harness/instrumentation-config.js"; +import { getOtelSettings } from "#harness/otel-settings.js"; import { recordErrorOnSpan } from "#internal/logging.js"; /** @@ -58,9 +58,8 @@ export interface TraceChannelRequestInput { * waiting for `event.waitUntil()` work or streamed response bodies. * * Emitting these spans is opt-in: unless authored instrumentation enables it - * via `defineInstrumentation({ traceChannelRequests: true })`, the handler runs - * with no span (`undefined`) and no context extraction — a true bypass, not a - * non-recording span. + * via `traceChannelRequests: true`, the handler runs with no span (`undefined`) + * and no context extraction — a true bypass, not a non-recording span. * * This is observability-only: it never changes the response and performs no * synchronous span export in the request path, adding only minimal in-process @@ -70,7 +69,7 @@ export async function traceChannelRequest( input: TraceChannelRequestInput, handler: (span: Span | undefined) => Promise, ): Promise { - if (getInstrumentationConfig()?.traceChannelRequests !== true) { + if (getOtelSettings()?.traceChannelRequests !== true) { return await handler(undefined); } diff --git a/packages/eve/src/public/instrumentation/otel.ts b/packages/eve/src/public/instrumentation/otel.ts new file mode 100644 index 0000000000..864db03dbb --- /dev/null +++ b/packages/eve/src/public/instrumentation/otel.ts @@ -0,0 +1,59 @@ +/** + * The OpenTelemetry authoring surface for `agent/instrumentation/`. + * + * Two halves, because OpenTelemetry has two: `otel()` is the settings a process + * can only hold one of, and an integration is a destination, of which there may + * be as many as there are files. + * + * Reachable only with `experimental.instrumentationProviders` on. With the flag + * off nothing discovers that directory, so these compile but never run. + */ + +import { createLocalTracesProcessor, resolveLocalTracesContent } from "#tracing/local-traces.js"; +import { contentFilteringProcessor } from "#tracing/content-span-processor.js"; +import { + agentRunsIntegration, + otelIntegration, + type ContentOptions, + type OtelIntegration, +} from "#tracing/otel-declaration.js"; + +export { + isOtelDeclaration, + isOtelIntegration, + otel, + otelIntegration, + type ContentOptions, + type OtelDeclaration, + type OtelIntegration, + type OtelIntegrationOptions, + type OtelOptions, +} from "#tracing/otel-declaration.js"; + +export type { SpanExporter, SpanProcessor } from "#compiled/@vercel/otel/index.js"; + +/** Vercel Agent Runs, enabled by default in production. */ +export function agentRuns(options: ContentOptions = {}): OtelIntegration { + return agentRunsIntegration(options); +} + +/** + * The local trace spool `eve dev` reads, as a destination. + * + * Export it from `agent/instrumentation/local.ts` to keep it alongside a hosted + * backend, or export `disableInstrumentation()` from that file to turn it off. + * Omitting the file leaves eve's default in place. + * `EVE_TRACES_CONTENT=off` narrows this destination only. + */ +export function localTraces(options: ContentOptions = {}): OtelIntegration { + const content = resolveLocalTracesContent(options); + const spool = createLocalTracesProcessor(); + return { + ...otelIntegration(), + content, + spanProcessors: + content.recordInputs && content.recordOutputs + ? [spool] + : [contentFilteringProcessor(spool, content)], + }; +} diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.ts index c135f11654..9b24609c8e 100644 --- a/packages/eve/src/tracing/agent-otel-provider.test.ts +++ b/packages/eve/src/tracing/agent-otel-provider.test.ts @@ -450,7 +450,8 @@ describe("createAgentOtelInstrumentation", () => { spanProcessors: [new SimpleSpanProcessor(exporter)], }); const agentOtel = createAgentOtelInstrumentation({ - captureContent: false, + recordInputs: false, + recordOutputs: false, frameworkVersion: "test", idGenerator, stateStore: new InMemoryAgentTraceStateStore(), diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.ts index 0e74fa277d..2607cc44bb 100644 --- a/packages/eve/src/tracing/agent-otel-provider.ts +++ b/packages/eve/src/tracing/agent-otel-provider.ts @@ -58,11 +58,11 @@ interface ToolSpanState extends SpanState { export interface AgentOtelInstrumentationInput { /** - * Capture model prompts/responses and tool call inputs/outputs as span - * attributes. Content stays on the local machine — this provider only - * wires the dev-time local spool — but can be turned off per project. + * The union of what this process's destinations requested. Each destination + * independently removes declined content before export. */ - readonly captureContent?: boolean; + readonly recordInputs?: boolean; + readonly recordOutputs?: boolean; readonly frameworkVersion: string; readonly idGenerator: AgentSpanIdGenerator; readonly stateStore: AgentTraceStateStore; @@ -79,7 +79,8 @@ export interface AgentOtelInstrumentation { export function createAgentOtelInstrumentation( input: AgentOtelInstrumentationInput, ): AgentOtelInstrumentation { - const captureContent = input.captureContent ?? true; + const recordInputs = input.recordInputs ?? true; + const recordOutputs = input.recordOutputs ?? true; const executionContexts = new WeakMap< InstrumentationAttemptScope, { readonly models: Map; readonly tools: Map } @@ -270,7 +271,7 @@ export function createAgentOtelInstrumentation( }, attempt.operation.context, ); - if (captureContent) { + if (recordInputs) { const messages = messagesContentAttribute(event.input.messages); if (messages !== undefined) span.setAttribute("ai.prompt.messages", messages); const system = systemPromptAttribute(event.input.instructions); @@ -291,7 +292,7 @@ export function createAgentOtelInstrumentation( setUsage(state.span, event.usage); const attempt = steps.get(event.scope); if (attempt !== undefined) setUsage(attempt.step.span, event.usage); - if (captureContent) { + if (recordOutputs) { state.span.setAttribute("ai.response.finish_reason", event.finishReason); const reasoning = textContentAttribute( event.content @@ -367,7 +368,7 @@ export function createAgentOtelInstrumentation( }, actionContext, ); - if (captureContent) { + if (recordInputs) { const args = contentAttribute(event.input, false); if (args !== undefined) toolSpan.setAttribute("gen_ai.tool.call.arguments", args); } @@ -390,7 +391,7 @@ export function createAgentOtelInstrumentation( } else if (event.output.type === "error") { recordError(state.toolSpan, event.output.error); recordError(state.span, event.output.error); - } else if (captureContent) { + } else if (recordOutputs) { const result = contentAttribute(event.output.output, false); if (result !== undefined) state.toolSpan.setAttribute("gen_ai.tool.call.result", result); } diff --git a/packages/eve/src/tracing/batch-span-processor.test.ts b/packages/eve/src/tracing/batch-span-processor.test.ts new file mode 100644 index 0000000000..e676280910 --- /dev/null +++ b/packages/eve/src/tracing/batch-span-processor.test.ts @@ -0,0 +1,171 @@ +import type { SpanExporter } from "#compiled/@vercel/otel/index.js"; +import { context, TraceFlags } from "@opentelemetry/api"; +import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks"; +import { isTracingSuppressed } from "@opentelemetry/core"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { batchSpanProcessor } from "#tracing/batch-span-processor.js"; + +interface RecordingExporter extends SpanExporter { + readonly batches: readonly (readonly unknown[])[]; +} + +function recordingExporter(behavior: { readonly fail?: boolean } = {}): RecordingExporter { + const batches: (readonly unknown[])[] = []; + return { + batches, + export: (spans, resultCallback) => { + batches.push([...spans]); + resultCallback( + behavior.fail === true ? { code: 1, error: new Error("refused") } : { code: 0 }, + ); + }, + shutdown: async () => undefined, + }; +} + +function span(index: number, traceFlags: TraceFlags = TraceFlags.SAMPLED): unknown { + return { index, spanContext: () => ({ traceFlags }) }; +} + +function batchIndexes(batches: readonly (readonly unknown[])[]): number[][] { + return batches.map((batch) => batch.map((item) => (item as { readonly index: number }).index)); +} + +describe("batchSpanProcessor", () => { + afterEach(() => { + vi.useRealTimers(); + context.disable(); + }); + + it("exports once the batch size is reached, without waiting for the timer", async () => { + const exporter = recordingExporter(); + const processor = batchSpanProcessor(exporter, { + maxExportBatchSize: 2, + scheduledDelayMillis: 60_000, + }); + + processor.onEnd(span(1)); + expect(exporter.batches).toHaveLength(0); + + processor.onEnd(span(2)); + await processor.forceFlush(); + + expect(batchIndexes(exporter.batches)).toStrictEqual([[1, 2]]); + }); + + it("splits a flush larger than one batch", async () => { + const exporter = recordingExporter(); + const processor = batchSpanProcessor(exporter, { maxExportBatchSize: 2 }); + + for (const index of [1, 2, 3]) processor.onEnd(span(index)); + await processor.forceFlush(); + + expect(batchIndexes(exporter.batches)).toStrictEqual([[1, 2], [3]]); + }); + + // An exporter that has stopped draining is already losing telemetry; taking + // the agent's memory with it would turn that into an outage. + it("drops spans past the queue limit rather than growing without bound", async () => { + const exporter = recordingExporter(); + const processor = batchSpanProcessor(exporter, { + maxExportBatchSize: 100, + maxQueueSize: 2, + }); + + for (const index of [1, 2, 3, 4]) processor.onEnd(span(index)); + await processor.forceFlush(); + + expect(batchIndexes(exporter.batches)).toStrictEqual([[1, 2]]); + }); + + it("logs a refused export rather than failing the turn that produced it", async () => { + const processor = batchSpanProcessor(recordingExporter({ fail: true }), { + maxExportBatchSize: 1, + }); + + processor.onEnd(span(1)); + await expect(processor.forceFlush()).resolves.toBeUndefined(); + }); + + it("exports only sampled spans", async () => { + const exporter = recordingExporter(); + const processor = batchSpanProcessor(exporter); + + processor.onEnd(span(1, TraceFlags.NONE)); + processor.onEnd(span(2)); + await processor.forceFlush(); + + expect(batchIndexes(exporter.batches)).toStrictEqual([[2]]); + }); + + it("suppresses tracing while an exporter runs", async () => { + const manager = new AsyncLocalStorageContextManager().enable(); + context.setGlobalContextManager(manager); + const suppressed: boolean[] = []; + const exporter: SpanExporter = { + export: (_spans, resultCallback) => { + suppressed.push(isTracingSuppressed(context.active())); + resultCallback({ code: 0 }); + }, + shutdown: async () => undefined, + }; + const processor = batchSpanProcessor(exporter); + + processor.onEnd(span(1)); + await processor.forceFlush(); + + expect(suppressed).toEqual([true]); + expect(isTracingSuppressed(context.active())).toBe(false); + manager.disable(); + }); + + it("does not overlap a later batch after the first times out", async () => { + vi.useFakeTimers(); + const batches: (readonly unknown[])[] = []; + const callbacks: Array<(result: { code: number }) => void> = []; + const exporter: SpanExporter = { + export: (spans, resultCallback) => { + batches.push([...spans]); + callbacks.push(resultCallback); + }, + shutdown: async () => undefined, + }; + const processor = batchSpanProcessor(exporter, { + exportTimeoutMillis: 10, + maxExportBatchSize: 1, + }); + + processor.onEnd(span(1)); + await vi.advanceTimersByTimeAsync(0); + processor.onEnd(span(2)); + const flushed = processor.forceFlush(); + let flushSettled = false; + void flushed.then(() => { + flushSettled = true; + }); + await vi.advanceTimersByTimeAsync(10); + + expect(batchIndexes(batches)).toStrictEqual([[1]]); + expect(flushSettled).toBe(true); + callbacks[0]?.({ code: 0 }); + await vi.advanceTimersByTimeAsync(0); + expect(batchIndexes(batches)).toStrictEqual([[1], [2]]); + callbacks[1]?.({ code: 0 }); + await flushed; + }); + + it("drains and then closes the exporter on shutdown, and takes nothing after", async () => { + const exporter = recordingExporter(); + const shutdown = vi.spyOn(exporter, "shutdown"); + const processor = batchSpanProcessor(exporter, { scheduledDelayMillis: 60_000 }); + + processor.onEnd(span(1)); + await processor.shutdown(); + processor.onEnd(span(2)); + await processor.forceFlush(); + + expect(batchIndexes(exporter.batches)).toStrictEqual([[1]]); + expect(shutdown).toHaveBeenCalledOnce(); + }); +}); diff --git a/packages/eve/src/tracing/batch-span-processor.ts b/packages/eve/src/tracing/batch-span-processor.ts new file mode 100644 index 0000000000..a30956f183 --- /dev/null +++ b/packages/eve/src/tracing/batch-span-processor.ts @@ -0,0 +1,179 @@ +import { TraceFlags, context, createContextKey } from "#compiled/@opentelemetry/api/index.js"; +import type { SpanExporter, SpanProcessor } from "#compiled/@vercel/otel/index.js"; + +import { createLogger, formatError } from "#internal/logging.js"; + +const log = createLogger("harness.batch-span-processor"); + +// This is the Symbol.for-backed key used by @opentelemetry/core's +// suppressTracing(), without pulling that package into eve's runtime. +const SUPPRESS_TRACING_KEY = createContextKey("OpenTelemetry SDK Context Key SUPPRESS_TRACING"); + +/** The OpenTelemetry specification's defaults for a batching processor. */ +const DEFAULTS = { + exportTimeoutMillis: 30_000, + maxExportBatchSize: 512, + maxQueueSize: 2048, + scheduledDelayMillis: 5_000, +} as const; + +/** @internal — exposed for tests; authors get the defaults. */ +export interface BatchSpanProcessorOptions { + readonly exportTimeoutMillis?: number; + readonly maxExportBatchSize?: number; + readonly maxQueueSize?: number; + readonly scheduledDelayMillis?: number; +} + +/** + * Buffers ended spans and hands them to one exporter in batches. + * + * eve owns this rather than taking `@opentelemetry/sdk-trace-base` as a runtime + * dependency: an exporter needs exactly one thing done for it, and the + * dependency would arrive with the whole SDK behind it. + * + * A full queue drops the newest span rather than growing without bound. An + * exporter that has stopped draining is already losing telemetry; taking the + * agent's memory with it would turn that into an outage. + */ +export function batchSpanProcessor( + exporter: SpanExporter, + options: BatchSpanProcessorOptions = {}, +): SpanProcessor { + const exportTimeoutMillis = options.exportTimeoutMillis ?? DEFAULTS.exportTimeoutMillis; + const maxExportBatchSize = options.maxExportBatchSize ?? DEFAULTS.maxExportBatchSize; + const maxQueueSize = options.maxQueueSize ?? DEFAULTS.maxQueueSize; + const scheduledDelayMillis = options.scheduledDelayMillis ?? DEFAULTS.scheduledDelayMillis; + + let buffer: unknown[] = []; + let timer: ReturnType | undefined; + // Exports are chained rather than raced so a flush cannot interleave two + // batches into the same exporter. + let pending: Promise = Promise.resolve(); + let stopped = false; + + function cancelTimer(): void { + if (timer === undefined) return; + clearTimeout(timer); + timer = undefined; + } + + function scheduleDrain(): void { + if (timer !== undefined) return; + timer = setTimeout(() => { + timer = undefined; + void drain(); + }, scheduledDelayMillis); + // Buffered spans must not be the reason a process stays alive; `shutdown` + // is what settles them. + timer.unref?.(); + } + + function drain(): Promise { + return waitForExporter(pendingDrain(), "span export").then(() => undefined); + } + + async function exportBatch(batch: readonly unknown[]): Promise { + try { + await new Promise((resolve, reject) => { + try { + context.with(context.active().setValue(SUPPRESS_TRACING_KEY, true), () => { + exporter.export(batch, (result) => { + if (result.code === 0) { + resolve(); + return; + } + reject(result.error ?? new Error("Span export failed.")); + }); + }); + } catch (error: unknown) { + reject(error instanceof Error ? error : new Error(String(error))); + } + }); + } catch (error: unknown) { + // A destination that cannot take spans is not a reason to fail the turn + // that produced them. + log.warn("span export failed", { error: formatError(error) }); + } + } + + return { + async forceFlush() { + if (stopped) return; + const drained = await waitForExporter(pendingDrain(), "span export"); + if (!drained || exporter.forceFlush === undefined) return; + pending = pending.then(() => settleExporter("exporter flush", exporter.forceFlush!)); + await waitForExporter(pending, "exporter flush"); + }, + onEnd(span) { + if (stopped) return; + if (!isSampled(span)) return; + if (buffer.length >= maxQueueSize) return; + + buffer.push(span); + if (buffer.length >= maxExportBatchSize) { + void drain(); + return; + } + scheduleDrain(); + }, + onStart() {}, + async shutdown() { + stopped = true; + const drained = await waitForExporter(pendingDrain(), "span export"); + buffer = []; + if (!drained) return; + pending = pending.then(() => settleExporter("exporter shutdown", exporter.shutdown)); + await waitForExporter(pending, "exporter shutdown"); + }, + }; + + function pendingDrain(): Promise { + cancelTimer(); + pending = pending.then(async () => { + while (buffer.length > 0) { + const batch = buffer.splice(0, maxExportBatchSize); + await exportBatch(batch); + } + }); + return pending; + } + + async function waitForExporter(operation: Promise, name: string): Promise { + let timer: ReturnType | undefined; + const timeout = new Promise((resolve) => { + timer = setTimeout(() => { + log.warn(`${name} timed out; preserving exporter serialization`, { + timeoutMillis: exportTimeoutMillis, + }); + resolve(false); + }, exportTimeoutMillis); + timer.unref?.(); + }); + const completed = operation.then(() => true); + try { + return await Promise.race([completed, timeout]); + } finally { + clearTimeout(timer); + } + } + + async function settleExporter(name: string, operation: () => Promise): Promise { + try { + await operation.call(exporter); + } catch (error: unknown) { + log.warn(`${name} failed`, { error: formatError(error) }); + } + } +} + +function isSampled(span: unknown): boolean { + if (typeof span !== "object" || span === null || !("spanContext" in span)) return false; + const spanContext = (span as { readonly spanContext?: unknown }).spanContext; + if (typeof spanContext !== "function") return false; + const context = Reflect.apply(spanContext, span, []) as { readonly traceFlags?: unknown }; + return ( + typeof context.traceFlags === "number" && + (context.traceFlags & TraceFlags.SAMPLED) === TraceFlags.SAMPLED + ); +} diff --git a/packages/eve/src/tracing/content-attributes.ts b/packages/eve/src/tracing/content-attributes.ts new file mode 100644 index 0000000000..7f4657eb31 --- /dev/null +++ b/packages/eve/src/tracing/content-attributes.ts @@ -0,0 +1,59 @@ +/** Prompts, instructions, and tool arguments. */ +const INPUT_CONTENT_ATTRIBUTES: ReadonlySet = new Set([ + "ai.documents", + "ai.prompt", + "ai.prompt.messages", + "ai.prompt.system", + "ai.prompt.toolChoice", + "ai.prompt.tools", + "ai.value", + "ai.values", + "gen_ai.input.messages", + "gen_ai.system_instructions", + "gen_ai.tool.call.arguments", + "gen_ai.tool.definitions", +]); + +/** Responses, reasoning, and tool results. */ +const OUTPUT_CONTENT_ATTRIBUTES: ReadonlySet = new Set([ + "ai.embedding", + "ai.embeddings", + "ai.ranking", + "ai.response.files", + "ai.response.object", + "ai.response.reasoning", + "ai.response.text", + "ai.response.toolCalls", + "ai.response.tool_calls", + "ai.response.tool_results", + "ai.toolCall.args", + "ai.toolCall.result", + "gen_ai.output.messages", + "gen_ai.tool.call.result", +]); + +/** What one destination is willing to receive. */ +export interface ResolvedContentOptions { + readonly recordInputs: boolean; + readonly recordOutputs: boolean; +} + +function isDeclined(key: string, content: ResolvedContentOptions): boolean { + if (!content.recordInputs && INPUT_CONTENT_ATTRIBUTES.has(key)) return true; + return !content.recordOutputs && OUTPUT_CONTENT_ATTRIBUTES.has(key); +} + +/** Returns a copy without declined content, or undefined when no copy is needed. */ +export function withoutDeclinedContent( + attributes: Readonly>, + content: ResolvedContentOptions, +): Record | undefined { + const keys = Object.keys(attributes); + if (!keys.some((key) => isDeclined(key, content))) return undefined; + + const kept: Record = {}; + for (const key of keys) { + if (!isDeclined(key, content)) kept[key] = attributes[key]; + } + return kept; +} diff --git a/packages/eve/src/tracing/content-span-processor.test.ts b/packages/eve/src/tracing/content-span-processor.test.ts new file mode 100644 index 0000000000..2da3459d0b --- /dev/null +++ b/packages/eve/src/tracing/content-span-processor.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it, vi } from "vitest"; + +import { contentFilteringProcessor } from "#tracing/content-span-processor.js"; + +describe("contentFilteringProcessor", () => { + it("gives one destination a redacted copy without mutating the shared span", () => { + const onEnd = vi.fn(); + const downstream = { + forceFlush: async () => undefined, + onEnd, + onStart: () => undefined, + shutdown: async () => undefined, + }; + const processor = contentFilteringProcessor(downstream, { + recordInputs: false, + recordOutputs: true, + }); + const span = { + attributes: { + "ai.prompt.messages": "secret", + "ai.response.text": "answer", + "service.name": "weather", + }, + spanContext: () => ({ traceFlags: 1 }), + }; + + processor.onEnd(span); + + expect(onEnd.mock.calls[0]?.[0].attributes).toStrictEqual({ + "ai.response.text": "answer", + "service.name": "weather", + }); + expect(span.attributes).toHaveProperty("ai.prompt.messages", "secret"); + }); +}); diff --git a/packages/eve/src/tracing/content-span-processor.ts b/packages/eve/src/tracing/content-span-processor.ts new file mode 100644 index 0000000000..b1c1b05873 --- /dev/null +++ b/packages/eve/src/tracing/content-span-processor.ts @@ -0,0 +1,45 @@ +import type { SpanProcessor } from "#compiled/@vercel/otel/index.js"; + +import { + withoutDeclinedContent, + type ResolvedContentOptions, +} from "#tracing/content-attributes.js"; +import { hasSessionRelease, type LocalTracesProcessor } from "#tracing/local-traces.js"; + +/** Gives one destination a redacted copy without mutating the shared span. */ +export function contentFilteringProcessor( + downstream: SpanProcessor, + content: ResolvedContentOptions, +): SpanProcessor { + const filtering: SpanProcessor = { + forceFlush: () => downstream.forceFlush(), + onEnd: (span) => { + downstream.onEnd(redacted(span, content)); + }, + onStart: (span, parentContext) => { + downstream.onStart(span, parentContext); + }, + shutdown: () => downstream.shutdown(), + }; + + if (!hasSessionRelease(downstream)) return filtering; + const releasing: LocalTracesProcessor = { + ...filtering, + releaseSession: (sessionId) => downstream.releaseSession(sessionId), + }; + return releasing; +} + +function redacted(span: unknown, content: ResolvedContentOptions): unknown { + if (typeof span !== "object" || span === null) return span; + + const attributes = (span as { readonly attributes?: unknown }).attributes; + if (typeof attributes !== "object" || attributes === null) return span; + + const kept = withoutDeclinedContent(attributes as Record, content); + if (kept === undefined) return span; + + return Object.create(span, { + attributes: { configurable: true, enumerable: true, value: kept }, + }); +} diff --git a/packages/eve/src/tracing/install-instrumentation-runtime.test.ts b/packages/eve/src/tracing/install-instrumentation-runtime.test.ts new file mode 100644 index 0000000000..2c3fef8a8b --- /dev/null +++ b/packages/eve/src/tracing/install-instrumentation-runtime.test.ts @@ -0,0 +1,61 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { installInstrumentationRuntime } from "#tracing/install-instrumentation-runtime.js"; +import { otelIntegration, collectOtelPipeline } from "#tracing/otel-declaration.js"; + +const { forceFlush, shutdown } = vi.hoisted(() => ({ + forceFlush: vi.fn(async () => undefined), + shutdown: vi.fn(async () => undefined), +})); + +vi.mock("#tracing/otel-registration.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + registerOtelPipeline: () => ({ + forceFlush, + idGenerator: { + allocateSpanId: () => "1".repeat(16), + withSpanId: (_spanId: string, run: () => unknown) => run(), + }, + shutdown, + }), + }; +}); + +vi.mock("#tracing/agent-otel-provider.js", () => ({ + createAgentOtelInstrumentation: () => ({ + hook: {}, + runInContext: (_operation: unknown, execute: () => PromiseLike) => execute(), + }), +})); + +const RUNTIME_GLOBAL_KEY = Symbol.for("eve.instrumentation-runtime"); + +describe("installInstrumentationRuntime", () => { + beforeEach(() => { + forceFlush.mockClear(); + shutdown.mockClear(); + delete (globalThis as Record)[RUNTIME_GLOBAL_KEY]; + }); + + it("flushes and shuts down the registered tracer provider", async () => { + const providerFlush = vi.fn(); + const providerShutdown = vi.fn(); + const runtime = installInstrumentationRuntime({ + collected: collectOtelPipeline([otelIntegration()]), + frameworkVersion: "test", + providers: [{ flush: providerFlush, shutdown: providerShutdown }], + serviceName: "weather", + }); + + await runtime.forceFlush(); + await runtime.shutdown(); + await runtime.shutdown(); + + expect(forceFlush).toHaveBeenCalledOnce(); + expect(providerFlush).toHaveBeenCalledOnce(); + expect(shutdown).toHaveBeenCalledOnce(); + expect(providerShutdown).toHaveBeenCalledOnce(); + }); +}); diff --git a/packages/eve/src/tracing/install-instrumentation-runtime.ts b/packages/eve/src/tracing/install-instrumentation-runtime.ts new file mode 100644 index 0000000000..9abc32eafb --- /dev/null +++ b/packages/eve/src/tracing/install-instrumentation-runtime.ts @@ -0,0 +1,99 @@ +import { trace } from "#compiled/@opentelemetry/api/index.js"; +import type { SpanProcessor } from "#compiled/@vercel/otel/index.js"; + +import { + createInstrumentationHooks, + type InstrumentationProviderDefinition, +} from "#harness/instrumentation-lifecycle.js"; +import { + registerInstrumentationRuntime, + type InstrumentationRuntime, +} from "#harness/instrumentation-runtime.js"; +import { activateOtelSettings } from "#harness/otel-settings.js"; +import { createLogger, formatError } from "#internal/logging.js"; +import { ContextAgentTraceStateStore } from "#tracing/agent-trace-context-store.js"; +import { createAgentOtelInstrumentation } from "#tracing/agent-otel-provider.js"; +import { hasSessionRelease, type LocalTracesProcessor } from "#tracing/local-traces.js"; +import type { CollectedOtel } from "#tracing/otel-declaration.js"; +import { registerOtelPipeline, type RegisteredOtelPipeline } from "#tracing/otel-registration.js"; + +const log = createLogger("tracing.install-instrumentation-runtime"); + +/** Installs the bus and the one OpenTelemetry pipeline collected for this process. */ +export function installInstrumentationRuntime(input: { + readonly collected: CollectedOtel; + readonly frameworkVersion: string; + readonly providers: readonly InstrumentationProviderDefinition[]; + readonly serviceName: string; +}): InstrumentationRuntime { + const providers: InstrumentationProviderDefinition[] = [...input.providers]; + let otelRuntime: RegisteredOtelPipeline | undefined; + let runInContext: InstrumentationRuntime["runInContext"] = (_operation, execute) => execute(); + + if (input.collected.declared) { + activateOtelSettings(input.collected.settings); + otelRuntime = registerOtelPipeline({ + pipeline: input.collected.pipeline, + serviceName: input.serviceName, + }); + const agentOtel = createAgentOtelInstrumentation({ + frameworkVersion: input.frameworkVersion, + idGenerator: otelRuntime.idGenerator, + recordInputs: input.collected.settings.recordInputs, + recordOutputs: input.collected.settings.recordOutputs, + stateStore: new ContextAgentTraceStateStore(), + tracer: trace.getTracer("eve.agent", input.frameworkVersion), + }); + providers.unshift(agentOtel.hook); + runInContext = agentOtel.runInContext; + + const releasable = input.collected.pipeline.spanProcessors + .filter(isSpanProcessor) + .filter(hasSessionRelease); + if (releasable.length > 0) providers.push(sessionReleaseProvider(releasable)); + } + + let shutdown: Promise | undefined; + return registerInstrumentationRuntime({ + forceFlush: () => + settleAll([ + ...(otelRuntime === undefined ? [] : [otelRuntime.forceFlush]), + ...providers.map((provider) => () => provider.flush?.()), + ]), + hooks: createInstrumentationHooks(providers), + runInContext, + shutdown: () => { + shutdown ??= settleAll([ + ...(otelRuntime === undefined ? [] : [otelRuntime.shutdown]), + ...providers.map((provider) => () => provider.shutdown?.()), + ]); + return shutdown; + }, + }); +} + +function isSpanProcessor(processor: SpanProcessor | "auto"): processor is SpanProcessor { + return processor !== "auto"; +} + +function sessionReleaseProvider( + processors: readonly LocalTracesProcessor[], +): InstrumentationProviderDefinition { + const release = async (event: { readonly sessionId: string }): Promise => { + await Promise.all(processors.map((processor) => processor.releaseSession(event.sessionId))); + }; + return { + events: { "session.completed": release, "session.failed": release }, + name: "eve.session-release", + }; +} + +/** One failing drain must not take the others or the step awaiting them with it. */ +async function settleAll(operations: readonly (() => void | PromiseLike)[]): Promise { + const results = await Promise.allSettled(operations.map(async (run) => run())); + for (const result of results) { + if (result.status === "rejected") { + log.warn("instrumentation drain failed", { error: formatError(result.reason) }); + } + } +} diff --git a/packages/eve/src/tracing/local-instrumentation-runtime.ts b/packages/eve/src/tracing/local-instrumentation-runtime.ts index ed340a4f44..5c4d8a5e19 100644 --- a/packages/eve/src/tracing/local-instrumentation-runtime.ts +++ b/packages/eve/src/tracing/local-instrumentation-runtime.ts @@ -1,19 +1,10 @@ -import { trace } from "#compiled/@opentelemetry/api/index.js"; - -import { ContextAgentTraceStateStore } from "#tracing/agent-trace-context-store.js"; -import { createAgentOtelInstrumentation } from "#tracing/agent-otel-provider.js"; -import { - createInstrumentationHooks, - type InstrumentationProviderDefinition, -} from "#harness/instrumentation-lifecycle.js"; import { getInstrumentationRuntime, - registerInstrumentationRuntime, type InstrumentationRuntime, } from "#harness/instrumentation-runtime.js"; -import { localTraces } from "#tracing/local-traces.js"; -import { mergeOtelDeclarations, otel } from "#tracing/otel-declaration.js"; -import { registerOtelPipeline } from "#tracing/otel-registration.js"; +import { installInstrumentationRuntime } from "#tracing/install-instrumentation-runtime.js"; +import { createLocalTracesProcessor } from "#tracing/local-traces.js"; +import { collectOtelPipeline, otel, otelIntegration } from "#tracing/otel-declaration.js"; /** Installs the zero-config local OTel runtime once in an `eve dev` worker. */ export function installLocalInstrumentationRuntime(input: { @@ -24,36 +15,26 @@ export function installLocalInstrumentationRuntime(input: { const existing = getInstrumentationRuntime(); if (existing !== undefined) return existing; - // The zero-config default expressed with the same primitive an authored - // `instrumentation.ts` would use, so this path exercises it. - const spool = localTraces({ appRoot: input.appRoot }); - const merged = mergeOtelDeclarations([otel({ spanProcessors: [spool] })]); - const idGenerator = registerOtelPipeline({ - options: merged ?? {}, - serviceName: input.serviceName, - }); - - const agentOtel = createAgentOtelInstrumentation({ - captureContent: process.env.EVE_TRACES_CONTENT !== "off", - frameworkVersion: input.frameworkVersion, - idGenerator, - stateStore: new ContextAgentTraceStateStore(), - tracer: trace.getTracer("eve.agent", input.frameworkVersion), - }); - const releaseTrace: InstrumentationProviderDefinition = { - events: { - "session.completed": releaseSessionTrace, - "session.failed": releaseSessionTrace, + // The zero-config default expressed with the same values an authored + // `agent/instrumentation/` would declare, so this path exercises them. + const spool = createLocalTracesProcessor({ appRoot: input.appRoot }); + const collectedPipeline = collectOtelPipeline([ + otel(), + otelIntegration({ spanProcessors: [spool] }), + ]); + const captureContent = process.env.EVE_TRACES_CONTENT !== "off"; + const collected = { + ...collectedPipeline, + settings: { + ...collectedPipeline.settings, + recordInputs: captureContent, + recordOutputs: captureContent, }, }; - - return registerInstrumentationRuntime({ - forceFlush: () => spool.forceFlush(), - hooks: createInstrumentationHooks([agentOtel.hook, releaseTrace]), - runInContext: agentOtel.runInContext, + return installInstrumentationRuntime({ + collected, + frameworkVersion: input.frameworkVersion, + providers: [], + serviceName: input.serviceName, }); - - async function releaseSessionTrace(event: { readonly sessionId: string }): Promise { - await spool.releaseSession(event.sessionId); - } } diff --git a/packages/eve/src/tracing/local-traces.test.ts b/packages/eve/src/tracing/local-traces.test.ts index 3ad8259ca1..fb8e1b50a4 100644 --- a/packages/eve/src/tracing/local-traces.test.ts +++ b/packages/eve/src/tracing/local-traces.test.ts @@ -1,6 +1,7 @@ -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; -import { localTraces } from "#tracing/local-traces.js"; +import { createLocalTracesProcessor } from "#tracing/local-traces.js"; +import { localTraces } from "#public/instrumentation/otel.js"; vi.mock("#tracing/local-trace-span-processor.js", () => ({ LocalTraceSpanProcessor: class { @@ -28,9 +29,13 @@ function agentSpan(sessionId: string, traceId: string): unknown { }; } -describe("localTraces", () => { +describe("createLocalTracesProcessor", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + it("reports whether the released session owned any traces", async () => { - const spool = localTraces({ appRoot: "/tmp/eve-local-traces-test" }); + const spool = createLocalTracesProcessor({ appRoot: "/tmp/eve-local-traces-test" }); spool.onStart(agentSpan("session-one", "a".repeat(32)), undefined); // A subagent child owns none, so releasing it leaves the trace pinned. @@ -41,10 +46,20 @@ describe("localTraces", () => { }); it("is a span processor, so it composes wherever one goes", () => { - const spool = localTraces({ appRoot: "/tmp/eve-local-traces-test" }); + const spool = createLocalTracesProcessor({ appRoot: "/tmp/eve-local-traces-test" }); expect(typeof spool.onStart).toBe("function"); expect(typeof spool.onEnd).toBe("function"); expect(typeof spool.forceFlush).toBe("function"); expect(typeof spool.shutdown).toBe("function"); }); + + it("is inert outside a development worker", async () => { + vi.stubEnv("EVE_DEV_WORKER_APP_ROOT", undefined); + const [processor] = localTraces().spanProcessors; + if (processor === undefined || processor === "auto") throw new Error("Expected a processor."); + + expect(() => processor.onEnd(agentSpan("session-one", "a".repeat(32)))).not.toThrow(); + await expect(processor.forceFlush()).resolves.toBeUndefined(); + await expect(processor.shutdown()).resolves.toBeUndefined(); + }); }); diff --git a/packages/eve/src/tracing/local-traces.ts b/packages/eve/src/tracing/local-traces.ts index 4b91fbe72b..5878005793 100644 --- a/packages/eve/src/tracing/local-traces.ts +++ b/packages/eve/src/tracing/local-traces.ts @@ -23,14 +23,27 @@ export interface LocalTracesProcessor extends SpanProcessor { releaseSession(sessionId: string): Promise; } +/** Whether a processor still exposes the local spool's session lifecycle. */ +export function hasSessionRelease(processor: SpanProcessor): processor is LocalTracesProcessor { + return typeof (processor as Partial).releaseSession === "function"; +} + /** * Writes the OTLP/JSON spool under `.eve/traces/v1`. * * `EVE_TRACES=off` removes the writer but keeps the processor: eve still has * to observe spans to track which session owns which trace. + * + * Internal because of `releaseSession`, which eve's runtime drives off session + * lifecycle. The authored surface is `localTraces()`, which wraps this in an + * `OtelIntegration`. */ -export function localTraces(input: { readonly appRoot?: string } = {}): LocalTracesProcessor { - const appRoot = input.appRoot ?? resolveAppRoot(); +export function createLocalTracesProcessor( + input: { readonly appRoot?: string } = {}, +): LocalTracesProcessor { + const appRoot = input.appRoot ?? process.env["EVE_DEV_WORKER_APP_ROOT"]; + if (appRoot === undefined) return inertLocalTracesProcessor(); + const retention = resolveLocalTraceRetentionSettings(); const processor = new AgentTraceSpanProcessor( retention.enabled ? [new LocalTraceSpanProcessor(appRoot)] : [], @@ -68,10 +81,27 @@ export function localTraces(input: { readonly appRoot?: string } = {}): LocalTra }; } -function resolveAppRoot(): string { - const appRoot = process.env["EVE_DEV_WORKER_APP_ROOT"]; - if (appRoot === undefined) { - throw new Error("EVE_DEV_WORKER_APP_ROOT is required for local tracing."); - } - return appRoot; +/** Intersects the local destination policy with its environment override. */ +export function resolveLocalTracesContent( + options: { + readonly recordInputs?: boolean; + readonly recordOutputs?: boolean; + } = {}, +): { readonly recordInputs: boolean; readonly recordOutputs: boolean } { + const enabled = process.env.EVE_TRACES_CONTENT !== "off"; + return { + recordInputs: enabled && options.recordInputs !== false, + recordOutputs: enabled && options.recordOutputs !== false, + }; +} + +/** A production-authored `localTraces()` has no local development store. */ +function inertLocalTracesProcessor(): LocalTracesProcessor { + return { + forceFlush: async () => undefined, + onEnd: () => undefined, + onStart: () => undefined, + releaseSession: async () => false, + shutdown: async () => undefined, + }; } diff --git a/packages/eve/src/tracing/otel-declaration.test.ts b/packages/eve/src/tracing/otel-declaration.test.ts index d75b6d25ed..5c677c3047 100644 --- a/packages/eve/src/tracing/otel-declaration.test.ts +++ b/packages/eve/src/tracing/otel-declaration.test.ts @@ -1,9 +1,16 @@ -import type { SpanProcessor } from "#compiled/@vercel/otel/index.js"; +import type { SpanExporter, SpanProcessor } from "#compiled/@vercel/otel/index.js"; import { describe, expect, it } from "vitest"; -import { isOtelDeclaration, mergeOtelDeclarations, otel } from "#tracing/otel-declaration.js"; +import { + agentRunsIntegration, + collectOtelPipeline, + isOtelDeclaration, + isOtelIntegration, + otel, + otelIntegration, +} from "#tracing/otel-declaration.js"; -/** The merge only ever moves processors, so a fresh no-op is identity enough. */ +/** Collection only ever moves processors, so a fresh no-op is identity enough. */ function processor(): SpanProcessor { return { forceFlush: async () => undefined, @@ -13,67 +20,120 @@ function processor(): SpanProcessor { }; } +function exporter(): SpanExporter { + return { + export: (_spans, resultCallback) => { + resultCallback({ code: 0 }); + }, + shutdown: async () => undefined, + }; +} + describe("otel", () => { - it("declares a pipeline without registering anything", () => { - const declaration = otel({ spanProcessors: [processor()] }); + it("declares settings without registering anything", () => { + const declaration = otel({ sampler: "always_on" }); expect(isOtelDeclaration(declaration)).toBe(true); expect(isOtelDeclaration({ options: {} })).toBe(false); }); }); -describe("mergeOtelDeclarations", () => { - it("is absent when nothing declared a pipeline", () => { - expect(mergeOtelDeclarations([])).toBeUndefined(); +describe("otelIntegration", () => { + it("passes declared processors through untouched", () => { + const first = processor(); + const integration = otelIntegration({ spanProcessors: [first] }); + + expect(isOtelIntegration(integration)).toBe(true); + expect(integration.spanProcessors).toStrictEqual([first]); + }); + + it("wraps an exporter in a batching processor, after any declared ones", () => { + const first = processor(); + const integration = otelIntegration({ + spanProcessors: [first], + traceExporter: exporter(), + }); + + expect(integration.spanProcessors).toHaveLength(2); + expect(integration.spanProcessors[0]).toBe(first); + }); +}); + +describe("agentRunsIntegration", () => { + it("uses Vercel's automatic processor by default", () => { + const integration = agentRunsIntegration(); + + expect(integration.content).toStrictEqual({ recordInputs: true, recordOutputs: true }); + expect(integration.spanProcessors).toStrictEqual(["auto"]); }); - it("concatenates span processors in declaration order", () => { + it("wraps the request-context transport when its content policy is narrowed", () => { + const integration = agentRunsIntegration({ recordInputs: false }); + + expect(integration.content).toStrictEqual({ recordInputs: false, recordOutputs: true }); + expect(integration.spanProcessors).toHaveLength(1); + expect(integration.spanProcessors[0]).not.toBe("auto"); + }); +}); + +describe("collectOtelPipeline", () => { + it("reports nothing declared when nothing did", () => { + expect(collectOtelPipeline([]).declared).toBe(false); + expect(collectOtelPipeline([{ not: "a declaration" }]).declared).toBe(false); + }); + + it("concatenates destinations in declaration order", () => { const [first, second, third] = [processor(), processor(), processor()]; - const merged = mergeOtelDeclarations([ - otel({ spanProcessors: [first, second] }), - otel({ spanProcessors: [third] }), + const collected = collectOtelPipeline([ + otelIntegration({ spanProcessors: [first, second] }), + otelIntegration({ spanProcessors: [third] }), otel(), ]); - expect(merged?.spanProcessors).toStrictEqual([first, second, third]); + expect(collected.declared).toBe(true); + expect(collected.pipeline.spanProcessors).toStrictEqual([first, second, third]); + }); + + it("is declared by a destination alone, with no otel() beside it", () => { + const collected = collectOtelPipeline([otelIntegration({ spanProcessors: [processor()] })]); + + expect(collected.declared).toBe(true); + expect(collected.settings).toStrictEqual({ + functionId: undefined, + recordInputs: true, + recordOutputs: true, + traceChannelRequests: false, + }); }); - it("carries a singleton declared exactly once", () => { - const merged = mergeOtelDeclarations([ - otel({ sampler: "always_on" }), - otel({ propagators: ["tracecontext"] }), - otel({ resource: { "service.version": "abc" } }), + it("carries the singletons onto the pipeline and the rest onto the settings", () => { + const collected = collectOtelPipeline([ + otel({ + functionId: "weather", + propagators: ["tracecontext"], + resource: { "service.version": "abc" }, + sampler: "always_on", + traceChannelRequests: true, + }), ]); - expect(merged).toMatchObject({ + expect(collected.pipeline).toMatchObject({ propagators: ["tracecontext"], resource: { "service.version": "abc" }, sampler: "always_on", }); + expect(collected.settings).toStrictEqual({ + functionId: "weather", + recordInputs: false, + recordOutputs: false, + traceChannelRequests: true, + }); }); // A process has one tracer provider, so letting the first declaration win // would silently discard the second — the failure this throw exists to stop. - it.each([ - { key: "resource", one: otel({ resource: { a: "1" } }), two: otel({ resource: { b: "2" } }) }, - { key: "sampler", one: otel({ sampler: "always_on" }), two: otel({ sampler: "always_off" }) }, - { - key: "propagators", - one: otel({ propagators: ["tracecontext"] }), - two: otel({ propagators: ["baggage"] }), - }, - ])("refuses a second $key rather than picking one", ({ key, one, two }) => { - expect(() => mergeOtelDeclarations([one, two])).toThrow( - new RegExp(`declares \`${key}\` in more than one`, "u"), - ); - }); - - it("names both declarations in the collision, so the author can find them", () => { + it("refuses a second otel() rather than picking one", () => { expect(() => - mergeOtelDeclarations([ - otel(), - otel({ sampler: "always_on" }), - otel({ sampler: "always_off" }), - ]), - ).toThrow(/\(1 and 2\)/u); + collectOtelPipeline([otel({ sampler: "always_on" }), otel({ sampler: "always_off" })]), + ).toThrow(/declares `otel\(\)` more than once/u); }); }); diff --git a/packages/eve/src/tracing/otel-declaration.ts b/packages/eve/src/tracing/otel-declaration.ts index fb1093fdd1..18fc5545b3 100644 --- a/packages/eve/src/tracing/otel-declaration.ts +++ b/packages/eve/src/tracing/otel-declaration.ts @@ -1,17 +1,40 @@ import type { PropagatorOrName, SamplerOrName, + SpanExporter, SpanProcessor, + SpanProcessorOrName, } from "#compiled/@vercel/otel/index.js"; +import { PROVIDER, type InstrumentationProvider } from "#public/instrumentation/provider.js"; +import { batchSpanProcessor } from "#tracing/batch-span-processor.js"; +import type { ResolvedContentOptions } from "#tracing/content-attributes.js"; +import { contentFilteringProcessor } from "#tracing/content-span-processor.js"; +import { vercelRuntimeSpanProcessor } from "#tracing/vercel-runtime-span-exporter.js"; + /** - * OpenTelemetry pipeline settings for one `otel()` declaration. + * The process-wide OpenTelemetry settings, declared by `otel()`. + * + * Everything here is a singleton, which is why it is one file: a process has + * one tracer provider, so it has one resource, one sampler, and one propagator + * set. Destinations are the plural half and live in `otelIntegration()`. * * `contextManager` and `instrumentations` are deliberately absent: eve's span * nesting depends on the former, and the latter cannot reach anything eve * imported before setup ran. */ export interface OtelOptions { + /** + * The function identifier attached to telemetry spans + * (`ai.telemetry.functionId`). Defaults to the agent name. + */ + readonly functionId?: string; + /** + * Whether to emit the inbound HTTP `SERVER` span that wraps each channel + * request — the parent of the turn trace and of any `hook.resume` or + * outgoing HTTP spans. Defaults to `false`. + */ + readonly traceChannelRequests?: boolean; /** * Resource attributes merged into eve's own, which already carry the * service name. @@ -26,27 +49,93 @@ export interface OtelOptions { readonly sampler?: SamplerOrName; /** Composed into one propagator. All inject; the first to extract wins. */ readonly propagators?: readonly PropagatorOrName[]; - /** - * Where eve exports this agent's traces, and the only field most agents set. - * Absent means nowhere, eve's own sinks included. - */ +} + +/** Where one `otelIntegration()` sends spans. */ +export interface OtelIntegrationOptions { + /** Merged into the pipeline in declaration order. */ readonly spanProcessors?: readonly SpanProcessor[]; + /** Wrapped in eve's batching processor and appended after `spanProcessors`. */ + readonly traceExporter?: SpanExporter; +} + +/** What one built-in destination records of the conversation itself. */ +export interface ContentOptions { + /** Record model prompts and tool call inputs. Defaults to `true`. */ + readonly recordInputs?: boolean; + /** Record model responses and tool call outputs. Defaults to `true`. */ + readonly recordOutputs?: boolean; } const OTEL_DECLARATION = Symbol.for("eve.instrumentation.otel"); +const OTEL_INTEGRATION = Symbol.for("eve.instrumentation.otel-integration"); /** - * One declared OpenTelemetry pipeline. eve collects every declaration before - * building the tracer provider, so this is a value rather than a side effect. + * The declared OpenTelemetry pipeline settings. eve collects this before + * building the tracer provider, so it is a value rather than a side effect. */ -export interface OtelDeclaration { +export interface OtelDeclaration extends InstrumentationProvider { readonly [OTEL_DECLARATION]: true; readonly options: OtelOptions; } -/** Declares the OpenTelemetry pipeline eve should build. */ +/** One declared destination. A process may have as many as it has files. */ +export interface OtelIntegration extends InstrumentationProvider { + readonly [OTEL_INTEGRATION]: true; + readonly content: ResolvedContentOptions; + readonly spanProcessors: readonly SpanProcessorOrName[]; +} + +/** + * Declares the process-wide OpenTelemetry settings. + * + * Export it from `agent/instrumentation/otel.ts`. Omitting the file is the + * common case: eve registers the pipeline for whatever destinations are + * declared beside it, and this only names what those destinations share. + */ export function otel(options: OtelOptions = {}): OtelDeclaration { - return { [OTEL_DECLARATION]: true, options }; + return { [OTEL_DECLARATION]: true, [PROVIDER]: true, options }; +} + +/** + * Declares one destination for this agent's traces. + * + * A `traceExporter` is wrapped in eve's batching processor, which is what makes + * the one-line form of a hosted backend enough. Pass `spanProcessors` instead + * when the destination needs its own batching, sampling, or filtering. + */ +export function otelIntegration(options: OtelIntegrationOptions = {}): OtelIntegration { + const declared = options.spanProcessors ?? []; + return { + [OTEL_INTEGRATION]: true, + [PROVIDER]: true, + content: { recordInputs: true, recordOutputs: true }, + spanProcessors: + options.traceExporter === undefined + ? declared + : [...declared, batchSpanProcessor(options.traceExporter)], + }; +} + +/** Vercel Agent Runs through the production request-context transport. @internal */ +export function agentRunsIntegration(options: ContentOptions = {}): OtelIntegration { + const content = resolveContentOptions(options); + return { + [OTEL_INTEGRATION]: true, + [PROVIDER]: true, + content, + spanProcessors: + content.recordInputs && content.recordOutputs + ? ["auto"] + : [contentFilteringProcessor(vercelRuntimeSpanProcessor(), content)], + }; +} + +export function resolveContentOptions(options: ContentOptions): ResolvedContentOptions { + return { + recordInputs: options.recordInputs !== false, + recordOutputs: options.recordOutputs !== false, + }; } export function isOtelDeclaration(value: unknown): value is OtelDeclaration { @@ -57,39 +146,94 @@ export function isOtelDeclaration(value: unknown): value is OtelDeclaration { ); } -/** The settings a process can only hold one of, so two declarations collide. */ -const SINGLETONS = ["resource", "sampler", "propagators"] as const; +export function isOtelIntegration(value: unknown): value is OtelIntegration { + return ( + typeof value === "object" && + value !== null && + (value as Partial)[OTEL_INTEGRATION] === true + ); +} + +/** The one pipeline a process can register. @internal */ +export interface OtelPipeline { + readonly propagators?: readonly PropagatorOrName[]; + readonly resource?: Readonly>; + readonly sampler?: SamplerOrName; + readonly spanProcessors: readonly SpanProcessorOrName[]; +} + +/** What the harness reads at turn time, as opposed to at registration. @internal */ +export interface OtelHarnessSettings { + readonly functionId?: string; + readonly traceChannelRequests: boolean; + /** + * What to materialize on spans at all. Each destination independently strips + * anything it declined before export. + */ + readonly recordInputs?: boolean; + readonly recordOutputs?: boolean; +} + +/** @internal */ +export interface CollectedOtel { + /** + * Whether anything declared OpenTelemetry. False means eve should leave the + * global tracer provider slot alone rather than register an empty pipeline. + */ + readonly declared: boolean; + readonly pipeline: OtelPipeline; + readonly settings: OtelHarnessSettings; +} /** - * Merges every declaration into the one pipeline a process can register. + * Folds the declared values into the one pipeline a process can register. + * + * Destinations concatenate in declaration order. The singletons cannot: two + * `otel()` values is a boot error rather than a silent win for whichever eve + * happened to visit first. With one declaration per file that collision needs + * two files both exporting `otel()`, which is the only way to reach it. * - * `spanProcessors` concatenate in declaration order. The rest cannot: one - * process has one tracer provider, so it has one resource, one sampler, and - * one propagator set. Two declarations of the same one is a boot error rather - * than a silent win for whichever eve happened to visit first. + * @internal */ -export function mergeOtelDeclarations( - declarations: readonly OtelDeclaration[], -): OtelOptions | undefined { - if (declarations.length === 0) return undefined; - - const spanProcessors: SpanProcessor[] = []; - const owners = new Map<(typeof SINGLETONS)[number], number>(); - let merged: OtelOptions = {}; - declarations.forEach(({ options }, index) => { - spanProcessors.push(...(options.spanProcessors ?? [])); - for (const key of SINGLETONS) { - if (options[key] === undefined) continue; - const owner = owners.get(key); - if (owner !== undefined) { - throw new Error( - `Instrumentation declares \`${key}\` in more than one \`otel()\` call (${String(owner)} and ${String(index)}). One process has one OpenTelemetry tracer provider, so it has one \`${key}\` — declare it once.`, - ); - } - owners.set(key, index); - merged = { ...merged, [key]: options[key] }; +export function collectOtelPipeline(values: readonly unknown[]): CollectedOtel { + const spanProcessors: SpanProcessorOrName[] = []; + let declaration: OtelDeclaration | undefined; + let declared = false; + let recordInputs = false; + let recordOutputs = false; + + for (const value of values) { + if (isOtelIntegration(value)) { + declared = true; + recordInputs ||= value.content.recordInputs; + recordOutputs ||= value.content.recordOutputs; + spanProcessors.push(...value.spanProcessors); + continue; + } + if (!isOtelDeclaration(value)) continue; + if (declaration !== undefined) { + throw new Error( + "Instrumentation declares `otel()` more than once. One process has one OpenTelemetry tracer provider, so it has one resource, one sampler, and one propagator set — declare them in a single `otel()`.", + ); } - }); + declared = true; + declaration = value; + } - return { ...merged, spanProcessors }; + const options = declaration?.options ?? {}; + return { + declared, + pipeline: { + propagators: options.propagators, + resource: options.resource, + sampler: options.sampler, + spanProcessors, + }, + settings: { + functionId: options.functionId, + recordInputs, + recordOutputs, + traceChannelRequests: options.traceChannelRequests === true, + }, + }; } diff --git a/packages/eve/src/tracing/otel-registration.scenario.test.ts b/packages/eve/src/tracing/otel-registration.scenario.test.ts index 6fa45471e4..204b34c81c 100644 --- a/packages/eve/src/tracing/otel-registration.scenario.test.ts +++ b/packages/eve/src/tracing/otel-registration.scenario.test.ts @@ -14,7 +14,7 @@ describe("registerOtelPipeline", () => { it("verifies tracer ownership when the sampler records nothing", () => { expect(() => registerOtelPipeline({ - options: { sampler: "always_off" }, + pipeline: { sampler: "always_off", spanProcessors: [] }, serviceName: "weather", }), ).not.toThrow(); @@ -31,7 +31,7 @@ describe("registerOtelPipeline", () => { expect(() => registerOtelPipeline({ - options: {}, + pipeline: { spanProcessors: [] }, serviceName: "weather", }), ).toThrow(/another runtime already owns the global propagator/u); @@ -41,7 +41,7 @@ describe("registerOtelPipeline", () => { const exporter = new InMemorySpanExporter(); const processor = new SimpleSpanProcessor(exporter); registerOtelPipeline({ - options: { spanProcessors: [processor] }, + pipeline: { spanProcessors: [processor] }, serviceName: "weather", }); diff --git a/packages/eve/src/tracing/otel-registration.test.ts b/packages/eve/src/tracing/otel-registration.test.ts index dc7ed032c8..9e49ca9760 100644 --- a/packages/eve/src/tracing/otel-registration.test.ts +++ b/packages/eve/src/tracing/otel-registration.test.ts @@ -12,10 +12,11 @@ describe("registerOtelPipeline", () => { expect(() => registerOtelPipeline({ - options: { + pipeline: { propagators: ["tracecontext"], resource: { "service.version": "abc" }, sampler: "always_on", + spanProcessors: [], }, serviceName: "weather", }), @@ -36,9 +37,9 @@ describe("registerOtelPipeline", () => { it("omits the sampler entirely rather than passing undefined", () => { registerOTel.mockImplementation(() => undefined); - expect(() => registerOtelPipeline({ options: {}, serviceName: "weather" })).toThrow( - /already owns the global tracer provider/u, - ); + expect(() => + registerOtelPipeline({ pipeline: { spanProcessors: [] }, serviceName: "weather" }), + ).toThrow(/already owns the global tracer provider/u); const configuration = registerOTel.mock.calls.at(-1)?.[0] as Record; expect("traceSampler" in configuration).toBe(false); @@ -57,7 +58,7 @@ describe("registerOtelPipeline", () => { expect(() => registerOtelPipeline({ - options: { spanProcessors: [downstream] }, + pipeline: { spanProcessors: [downstream] }, serviceName: "weather", }), ).toThrow(); @@ -78,11 +79,24 @@ describe("registerOtelPipeline", () => { expect(downstream.onEnd).toHaveBeenCalledExactlyOnceWith({ name: "agent.turn" }); }); + it("passes Vercel's automatic processor through", () => { + registerOTel.mockImplementation(() => undefined); + + expect(() => + registerOtelPipeline({ pipeline: { spanProcessors: ["auto"] }, serviceName: "weather" }), + ).toThrow(); + + const configuration = registerOTel.mock.calls.at(-1)?.[0] as { + spanProcessors: unknown[]; + }; + expect(configuration.spanProcessors[0]).toBe("auto"); + }); + it("throws when the registration never reached a processor", () => { registerOTel.mockImplementation(() => undefined); - expect(() => registerOtelPipeline({ options: {}, serviceName: "weather" })).toThrow( - /another runtime already owns/u, - ); + expect(() => + registerOtelPipeline({ pipeline: { spanProcessors: [] }, serviceName: "weather" }), + ).toThrow(/another runtime already owns/u); }); }); diff --git a/packages/eve/src/tracing/otel-registration.ts b/packages/eve/src/tracing/otel-registration.ts index d2b7616ed4..a5d4d7e99a 100644 --- a/packages/eve/src/tracing/otel-registration.ts +++ b/packages/eve/src/tracing/otel-registration.ts @@ -3,10 +3,11 @@ import { registerOTel, type Configuration, type SpanProcessor, + type SpanProcessorOrName, } from "#compiled/@vercel/otel/index.js"; import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js"; -import type { OtelOptions } from "#tracing/otel-declaration.js"; +import type { OtelPipeline } from "#tracing/otel-declaration.js"; const REGISTRATION_SPAN_NAME = "eve.otel.registration"; @@ -69,34 +70,36 @@ class PrivateSpanFilteringProcessor implements SpanProcessor { * propagator, then verifies both through the global APIs. */ export function registerOtelPipeline(input: { - readonly options: OtelOptions; + readonly pipeline: OtelPipeline; readonly serviceName: string; -}): AgentSpanIdGenerator { - const { options } = input; +}): RegisteredOtelPipeline { + const { pipeline } = input; const idGenerator = new AgentSpanIdGenerator(); const markerPropagator = new RegistrationMarkerPropagator(); const configuration: Configuration = { - attributes: options.resource, + attributes: pipeline.resource, autoDetectResources: false, idGenerator, // eve imports the model SDK before any of this runs, so an auto // instrumentation registered here could not patch it anyway. instrumentations: [], - propagators: [...(options.propagators ?? ["none"]), markerPropagator], + propagators: [...(pipeline.propagators ?? ["none"]), markerPropagator], serviceName: input.serviceName, - spanProcessors: [new PrivateSpanFilteringProcessor(options.spanProcessors ?? [])], + spanProcessors: pipeline.spanProcessors.map((processor) => + isSpanProcessor(processor) ? new PrivateSpanFilteringProcessor([processor]) : processor, + ), }; registerOTel( // Absent means "let `@vercel/otel` decide", which is not the same as // passing an explicit `undefined` sampler. - options.sampler === undefined + pipeline.sampler === undefined ? configuration - : { ...configuration, traceSampler: options.sampler }, + : { ...configuration, traceSampler: pipeline.sampler }, ); if (!globalTracerUses(idGenerator)) { throw new Error( - "eve could not register OpenTelemetry because another runtime already owns the global tracer provider. Remove the other `registerOTel` call, or move its exporters into eve's `otel({ spanProcessors: [...] })`.", + "eve could not register OpenTelemetry because another runtime already owns the global tracer provider. Remove the other `registerOTel` call, or move its exporters into eve's `otelIntegration({ spanProcessors: [...] })`.", ); } if (!markerPropagator.isInstalled()) { @@ -104,7 +107,33 @@ export function registerOtelPipeline(input: { "eve could not register OpenTelemetry because another runtime already owns the global propagator. Remove the other global propagator registration and declare propagators through eve's `otel()` instead.", ); } - return idGenerator; + const globalProvider = trace.getTracerProvider() as Partial; + const provider = (globalProvider.getDelegate?.() ?? + globalProvider) as Partial; + if (typeof provider.forceFlush !== "function" || typeof provider.shutdown !== "function") { + throw new Error("The registered OpenTelemetry tracer provider has no lifecycle methods."); + } + return { + forceFlush: () => provider.forceFlush!(), + idGenerator, + shutdown: () => provider.shutdown!(), + }; +} + +/** Lifecycle retained from the tracer provider that owns every destination. */ +export interface RegisteredOtelPipeline { + readonly forceFlush: () => Promise; + readonly idGenerator: AgentSpanIdGenerator; + readonly shutdown: () => Promise; +} + +interface RuntimeTracerProvider { + forceFlush(): Promise; + shutdown(): Promise; +} + +interface ProxyTracerProvider { + getDelegate(): unknown; } function globalTracerUses(idGenerator: AgentSpanIdGenerator): boolean { @@ -125,3 +154,7 @@ function isRegistrationSpan(span: unknown): boolean { span.name === REGISTRATION_SPAN_NAME ); } + +function isSpanProcessor(processor: SpanProcessorOrName): processor is SpanProcessor { + return processor !== "auto"; +} diff --git a/packages/eve/src/tracing/vercel-runtime-span-exporter.test.ts b/packages/eve/src/tracing/vercel-runtime-span-exporter.test.ts new file mode 100644 index 0000000000..f291fe9347 --- /dev/null +++ b/packages/eve/src/tracing/vercel-runtime-span-exporter.test.ts @@ -0,0 +1,42 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { vercelRuntimeSpanExporter } from "#tracing/vercel-runtime-span-exporter.js"; + +const { serializeRequest } = vi.hoisted(() => ({ + serializeRequest: vi.fn(() => new TextEncoder().encode('{"resourceSpans":[]}')), +})); + +vi.mock("#compiled/@opentelemetry/otlp-transformer/index.js", () => ({ + JsonTraceSerializer: { serializeRequest }, +})); + +const REQUEST_CONTEXT = Symbol.for("@vercel/request-context"); + +afterEach(() => { + delete (globalThis as Record)[REQUEST_CONTEXT]; + serializeRequest.mockClear(); +}); + +describe("vercelRuntimeSpanExporter", () => { + it("reports through the current Vercel request context", () => { + const reportSpans = vi.fn(); + (globalThis as Record)[REQUEST_CONTEXT] = { + get: () => ({ telemetry: { reportSpans } }), + }; + const result = vi.fn(); + + vercelRuntimeSpanExporter().export([{}], result); + + expect(reportSpans).toHaveBeenCalledExactlyOnceWith({ resourceSpans: [] }); + expect(result).toHaveBeenCalledExactlyOnceWith({ code: 0 }); + }); + + it("is a successful no-op without request telemetry", () => { + const result = vi.fn(); + + vercelRuntimeSpanExporter().export([{}], result); + + expect(serializeRequest).not.toHaveBeenCalled(); + expect(result).toHaveBeenCalledExactlyOnceWith({ code: 0 }); + }); +}); diff --git a/packages/eve/src/tracing/vercel-runtime-span-exporter.ts b/packages/eve/src/tracing/vercel-runtime-span-exporter.ts new file mode 100644 index 0000000000..49d1519baa --- /dev/null +++ b/packages/eve/src/tracing/vercel-runtime-span-exporter.ts @@ -0,0 +1,81 @@ +import { JsonTraceSerializer } from "#compiled/@opentelemetry/otlp-transformer/index.js"; +import { TraceFlags } from "#compiled/@opentelemetry/api/index.js"; +import type { SpanExporter, SpanProcessor } from "#compiled/@vercel/otel/index.js"; + +import { createLogger, formatError } from "#internal/logging.js"; + +const VERCEL_REQUEST_CONTEXT = Symbol.for("@vercel/request-context"); +const log = createLogger("tracing.vercel-runtime-span-exporter"); + +interface VercelRequestContextReader { + get(): + | { + readonly telemetry?: { readonly reportSpans: (data: unknown) => void }; + } + | undefined; +} + +/** The request-context transport behind @vercel/otel's automatic processor. */ +export function vercelRuntimeSpanExporter(): SpanExporter { + return { + export(spans, resultCallback) { + const reader = (globalThis as Record)[VERCEL_REQUEST_CONTEXT] as + | VercelRequestContextReader + | undefined; + const telemetry = reader?.get()?.telemetry; + if (telemetry === undefined) { + resultCallback({ code: 0 }); + return; + } + + try { + const serialized = JsonTraceSerializer.serializeRequest([...spans]); + if (serialized === undefined) throw new Error("Failed to serialize spans."); + telemetry.reportSpans(JSON.parse(new TextDecoder().decode(serialized)) as unknown); + resultCallback({ code: 0 }); + } catch (error: unknown) { + resultCallback({ + code: 1, + error: error instanceof Error ? error : new Error(String(error)), + }); + } + }, + forceFlush: async () => undefined, + shutdown: async () => undefined, + }; +} + +/** Reports immediately so the span stays attached to its request context. */ +export function vercelRuntimeSpanProcessor(): SpanProcessor { + const exporter = vercelRuntimeSpanExporter(); + let stopped = false; + return { + forceFlush: () => exporter.forceFlush?.() ?? Promise.resolve(), + onEnd(span) { + if (stopped || !isSampled(span)) return; + exporter.export([span], (result) => { + if (result.code !== 0) { + log.warn("Agent Runs export failed", { + error: formatError(result.error ?? new Error("Span export failed.")), + }); + } + }); + }, + onStart() {}, + async shutdown() { + stopped = true; + await exporter.shutdown(); + }, + }; +} + +function isSampled(span: unknown): boolean { + if (typeof span !== "object" || span === null || !("spanContext" in span)) return false; + const spanContext = (span as { readonly spanContext?: unknown }).spanContext; + if (typeof spanContext !== "function") return false; + const value = Reflect.apply(spanContext, span, []) as { readonly traceFlags?: unknown }; + return ( + typeof value.traceFlags === "number" && + (value.traceFlags & TraceFlags.SAMPLED) === TraceFlags.SAMPLED + ); +} diff --git a/packages/eve/test/scenarios/compiled-artifacts-bootstrap.scenario.test.ts b/packages/eve/test/scenarios/compiled-artifacts-bootstrap.scenario.test.ts index 314f620307..0659305653 100644 --- a/packages/eve/test/scenarios/compiled-artifacts-bootstrap.scenario.test.ts +++ b/packages/eve/test/scenarios/compiled-artifacts-bootstrap.scenario.test.ts @@ -177,9 +177,22 @@ describe("writeCompiledArtifactsFiles", () => { expect(instrumentationPluginSource).toContain('slot: "local"'); expect(instrumentationPluginSource).toContain('slot: "otel"'); + expect(instrumentationPluginSource).toContain("seedInstrumentationProviders();"); + expect(instrumentationPluginSource).toContain("shutdownInstrumentationProviders"); + expect(instrumentationPluginSource).toContain("hooks?.hook('close'"); expect(instrumentationPluginSource).not.toContain("registerInstrumentationConfig"); - await import(pathToFileURL(instrumentationPluginPath).href); + const instrumentationPlugin = (await import(pathToFileURL(instrumentationPluginPath).href)) as { + default: (nitroApp: { + hooks: { hook(name: "close", handler: () => Promise): void }; + }) => void; + }; + const closeHandlers: Array<() => Promise> = []; + instrumentationPlugin.default({ + hooks: { + hook: (_name, handler) => closeHandlers.push(handler), + }, + }); // The plugin resolves the registry by absolute path while the assertion // resolves it by package alias, so this also proves the globalThis rooting @@ -192,6 +205,45 @@ describe("writeCompiledArtifactsFiles", () => { "otel:compiled-artifacts-providers-test-agent", ]); expect(getInstrumentationProviders().map((entry) => entry.slot)).toEqual(["local", "otel"]); + expect(closeHandlers).toHaveLength(1); + await closeHandlers[0]?.(); + }); + + it("generates the provider plugin for built-in destinations without authored files", async () => { + const { agentRoot, appRoot } = await createAppRoot( + "eve-compiled-artifacts-default-providers-", + { + packageName: "compiled-artifacts-default-providers-test-agent", + }, + ); + const outDir = join(appRoot, ".workflow-build"); + await writeFile( + join(agentRoot, "agent.ts"), + [ + "export default {", + ' model: "openai/gpt-5.4",', + " experimental: { instrumentationProviders: true },", + "};", + "", + ].join("\n"), + ); + await writeFile(join(agentRoot, "instructions.md"), "You are a precise assistant.\n"); + + const compileResult = await compileAgent({ startPath: appRoot }); + const generatedArtifacts = await writeCompiledArtifactsFiles({ + compileResult, + defaultWorkflowWorld: "local", + outDir, + }); + const instrumentationPluginPath = generatedArtifacts.instrumentationPluginPath; + if (instrumentationPluginPath === undefined) { + throw new Error("Expected instrumentation plugin path to be generated."); + } + + expect(generatedArtifacts.instrumentationSourcePaths).toEqual([]); + expect(await readFile(instrumentationPluginPath, "utf8")).toContain( + "seedInstrumentationProviders();", + ); }); it("surfaces instrumentation import failures when the Nitro plugin module loads", async () => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d2f1f84bcf..194ed20c69 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -151,13 +151,13 @@ importers: version: 1.1.1(react@19.2.6) '@vercel/analytics': specifier: 2.0.1 - version: 2.0.1(@sveltejs/kit@2.61.1(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(next@16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(nuxt@4.4.6(d63bb97f4e3da4922a64b57be4708ff3))(react@19.2.6)(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vue@3.5.35(typescript@6.0.3)) + version: 2.0.1(@sveltejs/kit@2.63.0(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(next@16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(nuxt@4.4.6(d63bb97f4e3da4922a64b57be4708ff3))(react@19.2.6)(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vue@3.5.35(typescript@6.0.3)) '@vercel/geistdocs': specifier: 1.19.4 - version: 1.19.4(d8751cc86e2a2d4e1825d02781c39fd6) + version: 1.19.4(bfac3c4f3ad5892c9ab3875b1f065b4e) '@vercel/speed-insights': specifier: 2.0.0 - version: 2.0.0(@sveltejs/kit@2.61.1(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(next@16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(nuxt@4.4.6(d63bb97f4e3da4922a64b57be4708ff3))(react@19.2.6)(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vue@3.5.35(typescript@6.0.3)) + version: 2.0.0(@sveltejs/kit@2.63.0(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(next@16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(nuxt@4.4.6(d63bb97f4e3da4922a64b57be4708ff3))(react@19.2.6)(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vue@3.5.35(typescript@6.0.3)) '@vgpu/core': specifier: 0.0.7 version: 0.0.7 @@ -1273,6 +1273,9 @@ importers: '@opentelemetry/context-async-hooks': specifier: 'catalog:' version: 2.6.1(@opentelemetry/api@1.9.1) + '@opentelemetry/core': + specifier: 'catalog:' + version: 2.6.1(@opentelemetry/api@1.9.1) '@opentelemetry/otlp-transformer': specifier: 0.214.0 version: 0.214.0(@opentelemetry/api@1.9.1) @@ -21044,11 +21047,11 @@ snapshots: - rollup - supports-color - '@sveltejs/kit@2.61.1(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0))': + '@sveltejs/kit@2.61.1(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0))': dependencies: '@standard-schema/spec': 1.1.0 '@sveltejs/acorn-typescript': 1.0.11(acorn@8.17.0) - '@sveltejs/vite-plugin-svelte': 7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) + '@sveltejs/vite-plugin-svelte': 7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) '@types/cookie': 0.6.0 acorn: 8.17.0 cookie: 0.6.0 @@ -21060,17 +21063,16 @@ snapshots: set-cookie-parser: 3.1.2 sirv: 3.0.2 svelte: 5.56.1(@typescript-eslint/types@8.59.4) - vite: 7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0) + vite: 8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0) optionalDependencies: '@opentelemetry/api': 1.9.1 typescript: 6.0.3 - optional: true - '@sveltejs/kit@2.61.1(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0))': + '@sveltejs/kit@2.63.0(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0))': dependencies: '@standard-schema/spec': 1.1.0 '@sveltejs/acorn-typescript': 1.0.11(acorn@8.17.0) - '@sveltejs/vite-plugin-svelte': 7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) + '@sveltejs/vite-plugin-svelte': 7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) '@types/cookie': 0.6.0 acorn: 8.17.0 cookie: 0.6.0 @@ -21082,10 +21084,11 @@ snapshots: set-cookie-parser: 3.1.2 sirv: 3.0.2 svelte: 5.56.1(@typescript-eslint/types@8.59.4) - vite: 8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0) + vite: 7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0) optionalDependencies: '@opentelemetry/api': 1.9.1 typescript: 6.0.3 + optional: true '@sveltejs/kit@2.63.0(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0))': dependencies: @@ -21833,16 +21836,16 @@ snapshots: - supports-color - zod - '@vercel/agent-readability@0.5.0(@sveltejs/kit@2.61.1(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(@vercel/functions@3.8.0(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(h3@1.15.11)(next@16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))': + '@vercel/agent-readability@0.5.0(@sveltejs/kit@2.63.0(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(@vercel/functions@3.8.0(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(h3@1.15.11)(next@16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))': optionalDependencies: - '@sveltejs/kit': 2.61.1(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) + '@sveltejs/kit': 2.63.0(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) '@vercel/functions': 3.8.0(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)) h3: 1.15.11 next: 16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@vercel/analytics@2.0.1(@sveltejs/kit@2.61.1(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(next@16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(nuxt@4.4.6(d63bb97f4e3da4922a64b57be4708ff3))(react@19.2.6)(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vue@3.5.35(typescript@6.0.3))': + '@vercel/analytics@2.0.1(@sveltejs/kit@2.63.0(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(next@16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(nuxt@4.4.6(d63bb97f4e3da4922a64b57be4708ff3))(react@19.2.6)(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vue@3.5.35(typescript@6.0.3))': optionalDependencies: - '@sveltejs/kit': 2.61.1(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) + '@sveltejs/kit': 2.63.0(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) next: 16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) nuxt: 4.4.6(d63bb97f4e3da4922a64b57be4708ff3) react: 19.2.6 @@ -22027,7 +22030,7 @@ snapshots: etag: 1.8.1 fs-extra: 11.1.0 - '@vercel/geistdocs@1.19.4(d8751cc86e2a2d4e1825d02781c39fd6)': + '@vercel/geistdocs@1.19.4(bfac3c4f3ad5892c9ab3875b1f065b4e)': dependencies: '@ai-sdk/react': 3.0.193(react@19.2.6)(zod@4.4.3) '@clack/prompts': 0.11.0 @@ -22035,7 +22038,7 @@ snapshots: '@orama/tokenizers': 3.1.18 '@streamdown/cjk': 1.0.3(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@10.2.2))(react@19.2.6)(unified@11.0.5) '@streamdown/code': 1.1.1(react@19.2.6) - '@vercel/agent-readability': 0.5.0(@sveltejs/kit@2.61.1(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(@vercel/functions@3.8.0(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(h3@1.15.11)(next@16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)) + '@vercel/agent-readability': 0.5.0(@sveltejs/kit@2.63.0(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(@vercel/functions@3.8.0(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(h3@1.15.11)(next@16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)) '@vercel/oidc': 3.8.0 ai: 6.0.191(zod@4.4.3) class-variance-authority: 0.7.1 @@ -22333,7 +22336,7 @@ snapshots: ms: 2.1.3 picocolors: 1.1.1 tar-stream: 3.1.7 - undici: 7.28.0 + undici: 7.29.0 xdg-app-paths: 5.1.0 zod: 4.4.3 transitivePeerDependencies: @@ -22344,9 +22347,9 @@ snapshots: dependencies: zod: 4.4.3 - '@vercel/speed-insights@2.0.0(@sveltejs/kit@2.61.1(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(next@16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(nuxt@4.4.6(d63bb97f4e3da4922a64b57be4708ff3))(react@19.2.6)(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vue@3.5.35(typescript@6.0.3))': + '@vercel/speed-insights@2.0.0(@sveltejs/kit@2.63.0(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(next@16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(nuxt@4.4.6(d63bb97f4e3da4922a64b57be4708ff3))(react@19.2.6)(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vue@3.5.35(typescript@6.0.3))': optionalDependencies: - '@sveltejs/kit': 2.61.1(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) + '@sveltejs/kit': 2.63.0(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) next: 16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) nuxt: 4.4.6(d63bb97f4e3da4922a64b57be4708ff3) react: 19.2.6