Skip to content

Commit 2c03418

Browse files
RhysSullivanRhysSullivan
committed
add app auth failure boundary tests
1 parent fe5a4fd commit 2c03418

3 files changed

Lines changed: 319 additions & 0 deletions

File tree

‎apps/cloud/src/services/auth-tool-failures.node.test.ts‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
// ---------------------------------------------------------------------------
2+
// Cloud app auth failure propagation
3+
// ---------------------------------------------------------------------------
4+
//
5+
// Exercises the cloud HTTP API boundary:
6+
//
7+
// test -> HttpApiClient -> ProtectedCloudApi -> execution engine
8+
// -> sandbox code -> OpenAPI tool invocation
9+
//
10+
// The assertion is intentionally on the final execution payload, not the
11+
// plugin facade, so reviewers can see that model-visible tool results carry
12+
// auth guidance instead of an opaque internal tool error.
13+
// ---------------------------------------------------------------------------
14+
15+
import { describe, expect, it } from "@effect/vitest";
16+
import { Effect, Schema } from "effect";
17+
import { HttpApi, HttpApiClient, HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi";
18+
19+
import { ScopeId } from "@executor-js/sdk";
20+
import { makeOpenApiHttpApiTestAddSpecPayload } from "@executor-js/plugin-openapi/testing";
21+
22+
import { ProtectedCloudApi, asOrg } from "./__test-harness__/api-harness";
23+
24+
const PingGroup = HttpApiGroup.make("default", { topLevel: true }).add(
25+
HttpApiEndpoint.get("ping", "/ping", { success: Schema.Unknown }),
26+
);
27+
28+
const MissingAuthSourceApi = HttpApi.make("cloudAuthFailureSource").add(PingGroup);
29+
30+
type CloudApiShape = HttpApiClient.ForApi<typeof ProtectedCloudApi>;
31+
type EffectSuccess<T> = T extends Effect.Effect<infer A, unknown, unknown> ? A : never;
32+
type ExecuteResult = EffectSuccess<ReturnType<CloudApiShape["executions"]["execute"]>>;
33+
34+
const expectModelVisibleAuthFailure = (execution: ExecuteResult) => {
35+
expect(execution.status).toBe("completed");
36+
if (execution.status !== "completed") return;
37+
expect(execution.isError).toBe(false);
38+
expect(JSON.stringify(execution.structured)).not.toContain("Internal tool error");
39+
expect(JSON.stringify(execution.structured)).not.toContain("Internal Tool Error");
40+
expect(execution.structured).toMatchObject({
41+
status: "completed",
42+
result: {
43+
ok: false,
44+
error: {
45+
code: "credential_binding_missing",
46+
details: {
47+
category: "authentication",
48+
recovery: {
49+
createSecretTool: "executor.coreTools.secrets.create",
50+
secretsUrl: "https://executor.sh/secrets",
51+
},
52+
},
53+
},
54+
},
55+
});
56+
};
57+
58+
describe("cloud auth tool failures", () => {
59+
it.effect("cloud propagates missing credential binding as model-visible auth failure", () =>
60+
Effect.gen(function* () {
61+
const org = `org_${crypto.randomUUID()}`;
62+
const namespace = `auth_${crypto.randomUUID().replace(/-/g, "_")}`;
63+
const scopeId = ScopeId.make(org);
64+
65+
yield* asOrg(org, (client) =>
66+
client.openapi.addSpec({
67+
params: { scopeId },
68+
payload: {
69+
...makeOpenApiHttpApiTestAddSpecPayload(MissingAuthSourceApi, {
70+
namespace,
71+
headers: {
72+
Authorization: { kind: "secret", prefix: "Bearer " },
73+
},
74+
}),
75+
baseUrl: "https://api.example.test",
76+
},
77+
}),
78+
);
79+
80+
const execution = yield* asOrg(org, (client) =>
81+
client.executions.execute({
82+
payload: {
83+
code: [
84+
`const result = await tools.${namespace}.default.ping({});`,
85+
"return result;",
86+
].join("\n"),
87+
},
88+
}),
89+
);
90+
91+
expectModelVisibleAuthFailure(execution);
92+
}),
93+
);
94+
});

‎apps/local/src/server/auth-tool-failures.test.ts‎

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
// ---------------------------------------------------------------------------
2+
// Local app auth failure propagation
3+
// ---------------------------------------------------------------------------
4+
//
5+
// Exercises the local HTTP API boundary:
6+
//
7+
// test -> HttpApiClient -> in-process LocalApi -> execution engine
8+
// -> sandbox code -> OpenAPI tool invocation
9+
//
10+
// The assertion is intentionally on the final execution payload, not the
11+
// plugin facade, so reviewers can see that model-visible tool results carry
12+
// auth guidance instead of an opaque internal tool error.
13+
// ---------------------------------------------------------------------------
14+
15+
import { afterAll, beforeAll, describe, expect, it } from "@effect/vitest";
16+
import { randomBytes } from "node:crypto";
17+
import { mkdtempSync, rmSync } from "node:fs";
18+
import { tmpdir } from "node:os";
19+
import { join } from "node:path";
20+
21+
import { Effect, Layer, Schema } from "effect";
22+
import { FetchHttpClient, HttpRouter, HttpServer } from "effect/unstable/http";
23+
import {
24+
HttpApi,
25+
HttpApiBuilder,
26+
HttpApiClient,
27+
HttpApiEndpoint,
28+
HttpApiGroup,
29+
} from "effect/unstable/httpapi";
30+
31+
import { addGroup, observabilityMiddleware } from "@executor-js/api";
32+
import { CoreHandlers, ExecutionEngineService, ExecutorService } from "@executor-js/api/server";
33+
import { createExecutionEngine } from "@executor-js/execution";
34+
import { fileSecretsPlugin } from "@executor-js/plugin-file-secrets";
35+
import { openApiPlugin } from "@executor-js/plugin-openapi";
36+
import {
37+
OpenApiExtensionService,
38+
OpenApiGroup,
39+
OpenApiHandlers,
40+
} from "@executor-js/plugin-openapi/api";
41+
import { makeOpenApiHttpApiTestAddSpecPayload } from "@executor-js/plugin-openapi/testing";
42+
import { makeQuickJsExecutor } from "@executor-js/runtime-quickjs";
43+
import { Scope, ScopeId, collectTables, createExecutor } from "@executor-js/sdk";
44+
45+
import { ErrorCaptureLive } from "./observability";
46+
import { createSqliteFumaDb } from "./sqlite-fumadb";
47+
48+
const TEST_BASE_URL = "http://local.test";
49+
50+
const PingGroup = HttpApiGroup.make("default", { topLevel: true }).add(
51+
HttpApiEndpoint.get("ping", "/ping", { success: Schema.Unknown }),
52+
);
53+
54+
const MissingAuthSourceApi = HttpApi.make("localAuthFailureSource").add(PingGroup);
55+
56+
const TestApi = addGroup(OpenApiGroup);
57+
type TestApiShape =
58+
typeof TestApi extends HttpApi.HttpApi<infer _Id, infer Groups>
59+
? HttpApiClient.Client<Groups, never>
60+
: never;
61+
62+
interface Harness {
63+
readonly fetch: typeof globalThis.fetch;
64+
readonly scopeId: ScopeId;
65+
readonly dispose: () => Promise<void>;
66+
}
67+
68+
const startHarness = async (tmpDir: string): Promise<Harness> => {
69+
const scopeId = ScopeId.make(`test-${randomBytes(4).toString("hex")}`);
70+
const plugins = [
71+
openApiPlugin({ httpClientLayer: FetchHttpClient.layer }),
72+
fileSecretsPlugin({ directory: tmpDir }),
73+
] as const;
74+
const sqlite = await createSqliteFumaDb({
75+
tables: collectTables(plugins),
76+
namespace: "executor_local_auth_tool_failures_test",
77+
path: join(tmpDir, "data.db"),
78+
});
79+
80+
const executor = await Effect.runPromise(
81+
createExecutor({
82+
scopes: [
83+
Scope.make({
84+
id: scopeId,
85+
name: "test",
86+
createdAt: new Date(),
87+
}),
88+
],
89+
db: sqlite.db,
90+
plugins,
91+
onElicitation: "accept-all",
92+
}),
93+
);
94+
95+
const engine = createExecutionEngine({
96+
executor,
97+
codeExecutor: makeQuickJsExecutor(),
98+
});
99+
100+
const TestObservability = observabilityMiddleware(TestApi);
101+
const TestApiBase = HttpApiBuilder.layer(TestApi).pipe(
102+
Layer.provide(CoreHandlers),
103+
Layer.provide(OpenApiHandlers),
104+
Layer.provide(TestObservability),
105+
Layer.provide(ErrorCaptureLive),
106+
);
107+
108+
const { handler: webHandler, dispose: disposeHandler } = HttpRouter.toWebHandler(
109+
TestApiBase.pipe(
110+
Layer.provideMerge(Layer.succeed(OpenApiExtensionService)(executor.openapi)),
111+
Layer.provideMerge(Layer.succeed(ExecutorService)(executor)),
112+
Layer.provideMerge(Layer.succeed(ExecutionEngineService)(engine)),
113+
Layer.provideMerge(HttpServer.layerServices),
114+
Layer.provideMerge(Layer.succeed(HttpRouter.RouterConfig)({ maxParamLength: 1000 })),
115+
),
116+
);
117+
118+
return {
119+
fetch: ((input: RequestInfo | URL, init?: RequestInit) =>
120+
webHandler(
121+
input instanceof Request ? input : new Request(input, init),
122+
)) as typeof globalThis.fetch,
123+
scopeId,
124+
dispose: async () => {
125+
await Effect.runPromise(Effect.ignore(Effect.tryPromise(() => disposeHandler())));
126+
await Effect.runPromise(
127+
Effect.ignore(Effect.tryPromise(() => Effect.runPromise(executor.close()))),
128+
);
129+
await sqlite.close();
130+
},
131+
};
132+
};
133+
134+
const run = <A, E>(body: (client: TestApiShape) => Effect.Effect<A, E>): Effect.Effect<A, E> =>
135+
Effect.gen(function* () {
136+
const client = yield* HttpApiClient.make(TestApi, { baseUrl: TEST_BASE_URL });
137+
return yield* body(client);
138+
}).pipe(
139+
Effect.provide(
140+
FetchHttpClient.layer.pipe(
141+
Layer.provide(Layer.succeed(FetchHttpClient.Fetch)(harness.fetch)),
142+
),
143+
),
144+
) as Effect.Effect<A, E>;
145+
146+
type EffectSuccess<T> = T extends Effect.Effect<infer A, unknown, unknown> ? A : never;
147+
type ExecuteResult = EffectSuccess<ReturnType<TestApiShape["executions"]["execute"]>>;
148+
149+
const expectModelVisibleAuthFailure = (execution: ExecuteResult) => {
150+
expect(execution.status).toBe("completed");
151+
if (execution.status !== "completed") return;
152+
expect(execution.isError).toBe(false);
153+
expect(JSON.stringify(execution.structured)).not.toContain("Internal tool error");
154+
expect(JSON.stringify(execution.structured)).not.toContain("Internal Tool Error");
155+
expect(execution.structured).toMatchObject({
156+
status: "completed",
157+
result: {
158+
ok: false,
159+
error: {
160+
code: "credential_binding_missing",
161+
details: {
162+
category: "authentication",
163+
recovery: {
164+
createSecretTool: "executor.coreTools.secrets.create",
165+
secretsUrl: "https://executor.sh/secrets",
166+
},
167+
},
168+
},
169+
},
170+
});
171+
};
172+
173+
let tmpDir: string;
174+
let harness: Harness;
175+
176+
beforeAll(async () => {
177+
tmpDir = mkdtempSync(join(tmpdir(), "executor-local-auth-tool-failures-"));
178+
harness = await startHarness(tmpDir);
179+
});
180+
181+
afterAll(async () => {
182+
await harness.dispose();
183+
rmSync(tmpDir, { recursive: true, force: true });
184+
});
185+
186+
describe("local auth tool failures", () => {
187+
it.effect("local propagates missing credential binding as model-visible auth failure", () =>
188+
Effect.gen(function* () {
189+
const namespace = `auth_${randomBytes(4).toString("hex")}`;
190+
yield* run((client) =>
191+
client.openapi.addSpec({
192+
params: { scopeId: harness.scopeId },
193+
payload: {
194+
...makeOpenApiHttpApiTestAddSpecPayload(MissingAuthSourceApi, {
195+
namespace,
196+
headers: {
197+
Authorization: { kind: "secret", prefix: "Bearer " },
198+
},
199+
}),
200+
baseUrl: "https://api.example.test",
201+
},
202+
}),
203+
);
204+
205+
const execution = yield* run((client) =>
206+
client.executions.execute({
207+
payload: {
208+
code: [
209+
`const result = await tools.${namespace}.default.ping({});`,
210+
"return result;",
211+
].join("\n"),
212+
},
213+
}),
214+
);
215+
216+
expectModelVisibleAuthFailure(execution);
217+
}),
218+
);
219+
});

‎packages/plugins/mcp/src/sdk/plugin.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1516,6 +1516,12 @@ export const mcpPlugin = definePlugin((options?: McpPluginOptions) => {
15161516
);
15171517

15181518
if (Result.isFailure(resolved) && sd.transport === "stdio") {
1519+
if (Predicate.isTagged(resolved.failure, "McpAuthRequiredError")) {
1520+
return yield* new McpConnectionError({
1521+
transport: sd.transport,
1522+
message: resolved.failure.message,
1523+
});
1524+
}
15191525
return yield* Effect.fail(resolved.failure);
15201526
}
15211527

0 commit comments

Comments
 (0)