feat(eve): harden the instrumentation bus and wire providers to it

.changeset/silver-donkeys-shake.md

@@ -0,0 +1,9 @@
+---
+"eve": patch
+---
+
+Harden the instrumentation bus behind `experimental.instrumentationProviders`.
+Events carry replay-stable identity, ordinary tools expose both durable
+`action.*` and AI SDK `tool.call.*` boundaries, and provider state, abandonment,
+flush, and shutdown survive the worker lifecycle. Provider setup also receives
+the local eval run reference through `context.evaluation`.

packages/eve/src/evals/cli/eval.ts

@@ -1,8 +1,13 @@
+import { randomUUID } from "node:crypto";
 import { readFile } from "node:fs/promises";
 import { basename, join } from "node:path";
 
 import { loadDevelopmentEnvironmentFiles } from "#cli/dev/environment.js";
 import { shutdownActiveSandboxHandles } from "#execution/sandbox/active-handles.js";
+import {
+  EVE_EVALUATION_ENV_FLAG,
+  EVE_EVALUATION_RUN_ID_ENV,
+} from "#internal/application/dev-environment.js";
 import { resolveApplicationRoot } from "#internal/application/paths.js";
 import { createDevelopmentServer, type DevelopmentServer } from "#internal/nitro/host.js";
 import { createEvalClient } from "#evals/cli/eval-client.js";
@@ -141,6 +146,10 @@ export async function runEvalCommand(
         url: options.url,
       });
     } else {
+      // Set before the server boots, because a provider's `setup` reads it
+      // once at startup and never again.
+      process.env[EVE_EVALUATION_ENV_FLAG] = "1";
+      process.env[EVE_EVALUATION_RUN_ID_ENV] = randomUUID();
       devServer = createDevelopmentServer(appRoot, { host: "127.0.0.1", port: 0 });
       const started = await devServer.start();
       client = await createEvalClient({ kind: "local", url: started.url });

packages/eve/src/execution/node-step.ts

@@ -4,6 +4,7 @@ import type { Runtime, SessionCapabilities } from "#channel/types.js";
 import { dispatchDynamicModelEvent } from "#context/dynamic-model-lifecycle.js";
 import { createHarnessDelegationToolDefinition } from "#execution/delegation-tool.js";
 import type { HarnessToolDefinition } from "#harness/execute-tool.js";
+import { LOAD_SKILL_TOOL_NAME } from "#runtime/skills/fragment-context.js";
 import { createToolLoopHarness } from "#harness/tool-loop.js";
 import type { HandleEventFn, HarnessToolMap, StepFn } from "#harness/types.js";
 import { resolveInstalledPackageInfo } from "#internal/application/package.js";
@@ -260,6 +261,8 @@ function resolveHarnessToolDefinition(input: {
       rawExecute,
       scope: def.name,
     }),
+    frameworkAction:
+      isFrameworkTool && def.name === LOAD_SKILL_TOOL_NAME ? "load-skill" : undefined,
     inputSchema: def.inputSchema ?? UNSPECIFIED_INPUT_SCHEMA,
     name: def.name,
     approval: def.approval,

packages/eve/src/execution/workflow-steps.ts

@@ -24,6 +24,7 @@ import { BundleKey, ChannelKey } from "#runtime/sessions/runtime-context-keys.js
 import { runStep } from "#context/run-step.js";
 import { deserializeContext, serializeContext } from "#context/serialize.js";
 import { getHarnessEmissionState } from "#harness/emission.js";
+import { preserveSerializedInstrumentationState } from "#harness/instrumentation-state.js";
 import { preserveSerializedAgentTraceState } from "#tracing/agent-trace-context-store.js";
 import { readTurnSleepDurationMs } from "#harness/turn-sleep.js";
 import { isTurnCancellation, throwIfTurnAborted } from "#harness/turn-cancellation.js";
@@ -452,11 +453,15 @@ export async function turnStep(rawInput: TurnStepInput): Promise<DurableStepResu
   } catch (error) {
     if (!isTurnCancellation(error)) throw error;
     writer.releaseLock();
+    // Both carve-outs exist because the cancellation epilogue still publishes
+    // `turn.cancelled` downstream, and the open spans and provider state it
+    // needs to close were written by the step being discarded.
+    const interrupted = serializeContext(ctx);
     return {
       action: "cancelled",
-      serializedContext: preserveSerializedAgentTraceState(
-        input.serializedContext,
-        serializeContext(ctx),
+      serializedContext: preserveSerializedInstrumentationState(
+        preserveSerializedAgentTraceState(input.serializedContext, interrupted),
+        interrupted,
       ),
       sessionState: input.sessionState,
     };