diff --git a/packages/ai/src/api/transform-messages.ts b/packages/ai/src/api/transform-messages.ts index ec7035a38c..7ac3a946b1 100644 --- a/packages/ai/src/api/transform-messages.ts +++ b/packages/ai/src/api/transform-messages.ts @@ -62,11 +62,25 @@ function replaceImagesWithPlaceholder(content: (TextContent | ImageContent)[], p return result; } +function normalizeToolResultMediaData(content: (TextContent | ImageContent)[]): (TextContent | ImageContent)[] { + let changed = false; + const normalized = content.map((block) => { + if (block.type !== "image") return block; + const prefix = `data:${block.mimeType};base64,`; + if (!block.data.startsWith(prefix)) return block; + changed = true; + return { ...block, data: block.data.slice(prefix.length) }; + }); + return changed ? normalized : content; +} + function downgradeUnsupportedImages(messages: Message[], model: Model): Message[] { const supportsImages = model.input.includes("image"); const supportsVideo = model.input.includes("video"); if (supportsImages && supportsVideo) { - return messages; + return messages.map((msg) => + msg.role === "toolResult" ? { ...msg, content: normalizeToolResultMediaData(msg.content) } : msg, + ); } return messages.map((msg) => { @@ -86,7 +100,7 @@ function downgradeUnsupportedImages(messages: Message[], model } if (msg.role === "toolResult") { - let content = msg.content; + let content = normalizeToolResultMediaData(msg.content); if (!supportsVideo) { content = replaceMediaWithPlaceholder(content, NO_VIDEO_TOOL_PLACEHOLDER, (b) => isVideoMimeType(b.mimeType), diff --git a/packages/ai/test/issue-1260-tool-result-image-data-url.test.ts b/packages/ai/test/issue-1260-tool-result-image-data-url.test.ts new file mode 100644 index 0000000000..f66799da2e --- /dev/null +++ b/packages/ai/test/issue-1260-tool-result-image-data-url.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import { convertResponsesMessages } from "../src/providers/openai-responses-shared.ts"; +import type { Context, Model } from "../src/types.ts"; + +const model = { + id: "gpt-5.5", + name: "GPT-5.5", + provider: "openai-codex", + api: "openai-codex-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + reasoning: true, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1_000_000, + maxTokens: 128_000, +} as Model<"openai-codex-responses">; + +function zeroUsage() { + return { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }; +} + +function convertToolImage(data: string) { + const context: Context = { + messages: [ + { + role: "assistant", + content: [ + { + type: "toolCall", + id: "call_image|fc_image", + name: "image_tool", + arguments: {}, + }, + ], + api: "openai-codex-responses", + provider: "openai-codex", + model: "gpt-5.5", + usage: zeroUsage(), + stopReason: "toolUse", + timestamp: 1, + }, + { + role: "toolResult", + toolCallId: "call_image|fc_image", + toolName: "image_tool", + content: [{ type: "image", data, mimeType: "image/png" }], + isError: false, + timestamp: 2, + }, + ], + tools: [], + }; + + return convertResponsesMessages(model, context, new Set(["openai-codex"])); +} + +describe("issue #1260 Responses tool-result image data URLs", () => { + it("adds exactly one data URL prefix to raw base64", () => { + expect(convertToolImage("QUJD")).toMatchObject([ + { type: "function_call", call_id: "call_image", name: "image_tool" }, + { + type: "function_call_output", + call_id: "call_image", + output: [{ type: "input_image", detail: "auto", image_url: "data:image/png;base64,QUJD" }], + }, + ]); + }); + + it("preserves an already-prefixed canonical image data URL", () => { + const dataUrl = "data:image/png;base64,QUJD"; + expect(convertToolImage(dataUrl)).toMatchObject([ + { type: "function_call", call_id: "call_image", name: "image_tool" }, + { + type: "function_call_output", + call_id: "call_image", + output: [{ type: "input_image", detail: "auto", image_url: dataUrl }], + }, + ]); + }); +});