diff --git a/models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/AnthropicChatModel.java b/models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/AnthropicChatModel.java index 6b3ea00ec7..71b370c5b8 100644 --- a/models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/AnthropicChatModel.java +++ b/models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/AnthropicChatModel.java @@ -105,6 +105,7 @@ import org.springframework.ai.chat.observation.DefaultChatModelObservationConvention; import org.springframework.ai.chat.prompt.ChatOptions; import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.content.ContentPart; import org.springframework.ai.content.Media; import org.springframework.ai.model.tool.ToolCallingManager; import org.springframework.ai.observation.conventions.AiProvider; @@ -726,7 +727,7 @@ else if (requestOptions.getCacheOptions().isMultiBlockSystemCaching() && systemT userCacheControl = cacheResolver.resolve(MessageType.USER, combinedText); } - if (hasCitationDocs || hasMedia || userCacheControl != null) { + if (hasCitationDocs || hasMedia || userCacheControl != null || userMessage.hasInterleavedContent()) { List contentBlocks = new ArrayList<>(); // Prepend citation document blocks to the first user message @@ -736,19 +737,29 @@ else if (requestOptions.getCacheOptions().isMultiBlockSystemCaching() && systemT } } - String text = userMessage.getText(); - if (text != null && !text.isEmpty()) { - TextBlockParam.Builder textBlockBuilder = TextBlockParam.builder().text(text); - if (userCacheControl != null) { - textBlockBuilder.cacheControl(userCacheControl); - cacheResolver.useCacheBlock(); + // Walking the content parts preserves the caller's text/media + // ordering. + // For a message built in the flat form the parts are simply the text + // followed by the media, so this produces the same blocks as before. + List contentParts = userMessage.getContentParts(); + int cacheTextPartIndex = lastNonEmptyTextPartIndex(contentParts); + for (int p = 0; p < contentParts.size(); p++) { + ContentPart contentPart = contentParts.get(p); + if (contentPart instanceof ContentPart.TextPart textPart) { + if (textPart.text().isEmpty()) { + continue; + } + TextBlockParam.Builder textBlockBuilder = TextBlockParam.builder().text(textPart.text()); + // A cache prefix has to cover the whole content, so it is + // attached to the final text block rather than the first. + if (userCacheControl != null && p == cacheTextPartIndex) { + textBlockBuilder.cacheControl(userCacheControl); + cacheResolver.useCacheBlock(); + } + contentBlocks.add(ContentBlockParam.ofText(textBlockBuilder.build())); } - contentBlocks.add(ContentBlockParam.ofText(textBlockBuilder.build())); - } - - if (hasMedia) { - for (Media media : userMessage.getMedia()) { - contentBlocks.add(getContentBlockParamByMedia(media)); + else if (contentPart instanceof ContentPart.MediaPart mediaPart) { + contentBlocks.add(getContentBlockParamByMedia(mediaPart.media())); } } @@ -1347,6 +1358,20 @@ private WebSearchTool20260209 toSdkWebSearchTool(AnthropicWebSearchTool webSearc return sdkBuilder.build(); } + /** + * Index of the last content part that produces a text block, or -1 if there is none. + * Empty text parts are skipped when building blocks, so they cannot carry the cache + * control marker either. + */ + private static int lastNonEmptyTextPartIndex(List contentParts) { + for (int i = contentParts.size() - 1; i >= 0; i--) { + if (contentParts.get(i) instanceof ContentPart.TextPart textPart && !textPart.text().isEmpty()) { + return i; + } + } + return -1; + } + /** * Converts a Spring AI {@link Media} object to an Anthropic SDK * {@link ContentBlockParam}. Supports images (PNG, JPEG, GIF, WebP) and PDF diff --git a/models/spring-ai-bedrock-converse/src/main/java/org/springframework/ai/bedrock/converse/BedrockProxyChatModel.java b/models/spring-ai-bedrock-converse/src/main/java/org/springframework/ai/bedrock/converse/BedrockProxyChatModel.java index 49df9ac777..7935fd176f 100644 --- a/models/spring-ai-bedrock-converse/src/main/java/org/springframework/ai/bedrock/converse/BedrockProxyChatModel.java +++ b/models/spring-ai-bedrock-converse/src/main/java/org/springframework/ai/bedrock/converse/BedrockProxyChatModel.java @@ -99,6 +99,7 @@ import org.springframework.ai.chat.observation.DefaultChatModelObservationConvention; import org.springframework.ai.chat.prompt.ChatOptions; import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.content.ContentPart; import org.springframework.ai.content.Media; import org.springframework.ai.model.tool.ToolCallingManager; import org.springframework.ai.observation.conventions.AiProvider; @@ -288,14 +289,24 @@ ConverseRequest createRequest(Prompt prompt) { List contents = new ArrayList<>(); if (message instanceof UserMessage) { var userMessage = (UserMessage) message; - contents.add(ContentBlock.fromText(userMessage.getText())); - - if (!CollectionUtils.isEmpty(userMessage.getMedia())) { - List mediaContent = userMessage.getMedia() - .stream() - .map(this::mapMediaToContentBlock) - .toList(); - contents.addAll(mediaContent); + // Walking the content parts preserves the caller's text/media + // ordering, + // which Bedrock's content block list expresses directly. For a + // message + // built in the flat form the parts are the text followed by the + // media. + for (ContentPart contentPart : userMessage.getContentParts()) { + if (contentPart instanceof ContentPart.TextPart textPart) { + // Bedrock rejects an empty text block, so blank text is + // skipped + // rather than sent as an empty content block. + if (StringUtils.hasText(textPart.text())) { + contents.add(ContentBlock.fromText(textPart.text())); + } + } + else if (contentPart instanceof ContentPart.MediaPart mediaPart) { + contents.add(mapMediaToContentBlock(mediaPart.media())); + } } } diff --git a/models/spring-ai-google-genai/src/main/java/org/springframework/ai/google/genai/GoogleGenAiChatModel.java b/models/spring-ai-google-genai/src/main/java/org/springframework/ai/google/genai/GoogleGenAiChatModel.java index e75999a853..60a799df46 100644 --- a/models/spring-ai-google-genai/src/main/java/org/springframework/ai/google/genai/GoogleGenAiChatModel.java +++ b/models/spring-ai-google-genai/src/main/java/org/springframework/ai/google/genai/GoogleGenAiChatModel.java @@ -77,6 +77,7 @@ import org.springframework.ai.chat.observation.ChatModelObservationDocumentation; import org.springframework.ai.chat.observation.DefaultChatModelObservationConvention; import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.content.ContentPart; import org.springframework.ai.content.Media; import org.springframework.ai.google.genai.cache.GoogleGenAiCachedContentService; import org.springframework.ai.google.genai.common.GoogleGenAiConstants; @@ -247,6 +248,25 @@ List messageToGeminiParts(Message message) { return parts; } else if (message instanceof UserMessage userMessage) { + // Ordered content maps one-to-one onto Gemini's part list, so a prompt that + // interleaves text and media survives verbatim. Never empty for a message + // carrying any text or media, so the branch below is only reached for a + // message + // with neither. + List contentParts = userMessage.getContentParts(); + if (!contentParts.isEmpty()) { + List parts = new ArrayList<>(contentParts.size()); + for (ContentPart contentPart : contentParts) { + if (contentPart instanceof ContentPart.TextPart textPart) { + parts.add(Part.fromText(textPart.text())); + } + else if (contentPart instanceof ContentPart.MediaPart mediaPart) { + parts.add(mediaToPart(mediaPart.media())); + } + } + return parts; + } + List parts = new ArrayList<>(); if (userMessage.getText() != null) { parts.add(Part.fromText(userMessage.getText())); @@ -319,31 +339,25 @@ else if (message instanceof ToolResponseMessage toolResponseMessage) { } } - private static List mediaToParts(Collection media) { - List parts = new ArrayList<>(); - - List mediaParts = media.stream().map(mediaData -> { - Object data = mediaData.getData(); - String mimeType = mediaData.getMimeType().toString(); + private static Part mediaToPart(Media media) { + Object data = media.getData(); + String mimeType = media.getMimeType().toString(); - if (data instanceof byte[]) { - return Part.fromBytes((byte[]) data, mimeType); - } - else if (data instanceof URI || data instanceof String) { - // Handle URI or String URLs - String uri = data.toString(); - return Part.fromUri(uri, mimeType); - } - else { - throw new IllegalArgumentException("Unsupported media data type: " + data.getClass()); - } - }).toList(); - - if (!CollectionUtils.isEmpty(mediaParts)) { - parts.addAll(mediaParts); + if (data instanceof byte[]) { + return Part.fromBytes((byte[]) data, mimeType); + } + else if (data instanceof URI || data instanceof String) { + // Handle URI or String URLs + String uri = data.toString(); + return Part.fromUri(uri, mimeType); } + else { + throw new IllegalArgumentException("Unsupported media data type: " + data.getClass()); + } + } - return parts; + private static List mediaToParts(Collection media) { + return media.stream().map(GoogleGenAiChatModel::mediaToPart).toList(); } // Helper methods for JSON/Map conversion diff --git a/models/spring-ai-google-genai/src/test/java/org/springframework/ai/google/genai/CreateGeminiRequestTests.java b/models/spring-ai-google-genai/src/test/java/org/springframework/ai/google/genai/CreateGeminiRequestTests.java index bc818a651f..010039011b 100644 --- a/models/spring-ai-google-genai/src/test/java/org/springframework/ai/google/genai/CreateGeminiRequestTests.java +++ b/models/spring-ai-google-genai/src/test/java/org/springframework/ai/google/genai/CreateGeminiRequestTests.java @@ -33,6 +33,7 @@ import org.springframework.ai.chat.messages.SystemMessage; import org.springframework.ai.chat.messages.UserMessage; import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.content.ContentPart; import org.springframework.ai.content.Media; import org.springframework.ai.google.genai.GoogleGenAiChatModel.GeminiRequest; import org.springframework.ai.google.genai.common.GoogleGenAiServiceTier; @@ -117,6 +118,90 @@ public void createRequestWithSystemMessage() { assertThat(mediaPart.fileData().isPresent()).isTrue(); } + @Test + public void createRequestWithInterleavedContentParts() { + + var image1 = Media.builder() + .mimeType(MimeTypeUtils.IMAGE_JPEG) + .data(URI.create("http://example.com/page1.jpg")) + .build(); + var image2 = Media.builder() + .mimeType(MimeTypeUtils.IMAGE_JPEG) + .data(URI.create("http://example.com/page2.jpg")) + .build(); + + // A per-page prompt: each image must stay directly after its own page marker. + var userMessage = UserMessage.builder() + .contentParts(ContentPart.text("--- page 1 ---"), ContentPart.media(image1), + ContentPart.text("--- page 2 ---"), ContentPart.media(image2)) + .build(); + + var client = GoogleGenAiChatModel.builder().genAiClient(this.genAiClient).build(); + + GeminiRequest request = client.createGeminiRequest( + new Prompt(List.of(userMessage), GoogleGenAiChatOptions.builder().model("DEFAULT_MODEL").build())); + + assertThat(request.contents()).hasSize(1); + List parts = request.contents().get(0).parts().orElse(List.of()); + assertThat(parts).hasSize(4); + + assertThat(parts.get(0).text().orElse("")).isEqualTo("--- page 1 ---"); + assertThat(parts.get(1).fileData()).isPresent(); + assertThat(parts.get(1).fileData().get().fileUri().orElse("")).isEqualTo("http://example.com/page1.jpg"); + assertThat(parts.get(2).text().orElse("")).isEqualTo("--- page 2 ---"); + assertThat(parts.get(3).fileData()).isPresent(); + assertThat(parts.get(3).fileData().get().fileUri().orElse("")).isEqualTo("http://example.com/page2.jpg"); + } + + @Test + public void createRequestWithContentPartsAndSystemMessage() { + + var systemMessage = new SystemMessage("System Message Text"); + var userMessage = UserMessage.builder() + .contentParts(ContentPart.text("first"), ContentPart.text("second")) + .build(); + + var client = GoogleGenAiChatModel.builder().genAiClient(this.genAiClient).build(); + + GeminiRequest request = client.createGeminiRequest(new Prompt(List.of(systemMessage, userMessage), + GoogleGenAiChatOptions.builder().model("DEFAULT_MODEL").build())); + + // The system message is still hoisted out of contents into systemInstruction. + assertThat(request.config().systemInstruction()).isPresent(); + assertThat(request.config().systemInstruction().get().parts().get().get(0).text().orElse("")) + .isEqualTo("System Message Text"); + + assertThat(request.contents()).hasSize(1); + List parts = request.contents().get(0).parts().orElse(List.of()); + assertThat(parts).hasSize(2); + assertThat(parts.get(0).text().orElse("")).isEqualTo("first"); + assertThat(parts.get(1).text().orElse("")).isEqualTo("second"); + } + + @Test + public void createRequestWithLegacyTextAndMediaIsUnchanged() { + + // Pins the flat text-plus-media path: text first, then all media. A regression in + // how content parts are derived would show up here. + var userMessage = UserMessage.builder() + .text("User Message Text") + .media(List + .of(Media.builder().mimeType(MimeTypeUtils.IMAGE_PNG).data(URI.create("http://example.com")).build())) + .build(); + + var client = GoogleGenAiChatModel.builder().genAiClient(this.genAiClient).build(); + + GeminiRequest request = client.createGeminiRequest( + new Prompt(List.of(userMessage), GoogleGenAiChatOptions.builder().model("DEFAULT_MODEL").build())); + + assertThat(request.contents()).hasSize(1); + List parts = request.contents().get(0).parts().orElse(List.of()); + assertThat(parts).hasSize(2); + assertThat(parts.get(0).text().orElse("")).isEqualTo("User Message Text"); + assertThat(parts.get(1).fileData()).isPresent(); + assertThat(parts.get(1).fileData().get().fileUri().orElse("")).isEqualTo("http://example.com"); + } + @Test public void promptOptionsTools() { diff --git a/models/spring-ai-mistral-ai/src/main/java/org/springframework/ai/mistralai/MistralAiChatModel.java b/models/spring-ai-mistral-ai/src/main/java/org/springframework/ai/mistralai/MistralAiChatModel.java index 6d2b1fd484..79aa02eb04 100644 --- a/models/spring-ai-mistral-ai/src/main/java/org/springframework/ai/mistralai/MistralAiChatModel.java +++ b/models/spring-ai-mistral-ai/src/main/java/org/springframework/ai/mistralai/MistralAiChatModel.java @@ -16,6 +16,7 @@ package org.springframework.ai.mistralai; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -49,6 +50,7 @@ import org.springframework.ai.chat.observation.ChatModelObservationDocumentation; import org.springframework.ai.chat.observation.DefaultChatModelObservationConvention; import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.content.ContentPart; import org.springframework.ai.content.Media; import org.springframework.ai.mistralai.api.MistralAiApi; import org.springframework.ai.mistralai.api.MistralAiApi.ChatCompletion; @@ -467,20 +469,26 @@ private ChatCompletionMessage createSystemChatCompletionMessage(Message message) } private ChatCompletionMessage createUserChatCompletionMessage(Message message) { - var content = message.getText(); - Assert.state(content != null, "content must not be null"); - - if (message instanceof UserMessage userMessage && !CollectionUtils.isEmpty(userMessage.getMedia())) { - // @formatter:off - var contentChunks = Stream.concat( - Stream.of(new ChatCompletionMessage.TextChunk(content)), - this.mapToImageUrlChunks(userMessage) - ).toList(); - // @formatter:on + // Ordered content is handled before the non-null text assertion below, since a + // message whose content starts with media has no leading text to assert on. + if (message instanceof UserMessage userMessage + && (userMessage.hasInterleavedContent() || !CollectionUtils.isEmpty(userMessage.getMedia()))) { + var contentChunks = new ArrayList(); + for (ContentPart contentPart : userMessage.getContentParts()) { + if (contentPart instanceof ContentPart.TextPart textPart) { + contentChunks.add(new ChatCompletionMessage.TextChunk(textPart.text())); + } + else if (contentPart instanceof ContentPart.MediaPart mediaPart) { + contentChunks.add(this.mapToImageUrlChunk(mediaPart.media())); + } + } return new ChatCompletionMessage(contentChunks, ChatCompletionMessage.Role.USER); } + var content = message.getText(); + Assert.state(content != null, "content must not be null"); + return new ChatCompletionMessage(content, ChatCompletionMessage.Role.USER); } @@ -490,10 +498,6 @@ private ToolCall mapToolCall(AssistantMessage.ToolCall toolCall) { return new ToolCall(toolCall.id(), toolCall.type(), function, null); } - private Stream mapToImageUrlChunks(UserMessage userMessage) { - return userMessage.getMedia().stream().map(this::mapToImageUrlChunk); - } - private ChatCompletionMessage.ImageUrlChunk mapToImageUrlChunk(Media media) { return new ChatCompletionMessage.ImageUrlChunk(this.fromMediaData(media.getMimeType(), media.getData())); } diff --git a/models/spring-ai-mistral-ai/src/test/java/org/springframework/ai/mistralai/MistralAiChatCompletionRequestTests.java b/models/spring-ai-mistral-ai/src/test/java/org/springframework/ai/mistralai/MistralAiChatCompletionRequestTests.java index eb7bea6942..964bb43515 100644 --- a/models/spring-ai-mistral-ai/src/test/java/org/springframework/ai/mistralai/MistralAiChatCompletionRequestTests.java +++ b/models/spring-ai-mistral-ai/src/test/java/org/springframework/ai/mistralai/MistralAiChatCompletionRequestTests.java @@ -113,8 +113,7 @@ void chatCompletionRequestWithOptionsTest() { @Test void createChatCompletionMessagesWithUserMessage() { - var userMessage = new UserMessage(TEXT_CONTENT); - userMessage.getMedia().add(IMAGE_MEDIA); + var userMessage = UserMessage.builder().text(TEXT_CONTENT).media(IMAGE_MEDIA).build(); var prompt = createPrompt(userMessage); var chatCompletionRequest = this.chatModel.createRequest(prompt, false); verifyUserChatCompletionMessages(chatCompletionRequest.messages()); diff --git a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatModel.java b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatModel.java index 245fe6c8d4..3b7b3d40b4 100644 --- a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatModel.java +++ b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatModel.java @@ -375,6 +375,13 @@ OllamaApi.ChatRequest ollamaChatRequest(Prompt prompt, boolean stream) { return List.of(OllamaApi.Message.builder(Role.SYSTEM).content(message.getText()).build()); } else if (message.getMessageType() == MessageType.USER) { + // Ollama's message format is a text plus a separate image list, so it + // cannot + // express an ordering between the two. Ordered content parts are + // therefore + // flattened to text-then-images via getText()/getMedia(), which is the + // most + // this wire format can carry. var messageBuilder = OllamaApi.Message.builder(Role.USER).content(message.getText()); if (message instanceof UserMessage userMessage) { if (!CollectionUtils.isEmpty(userMessage.getMedia())) { diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java index e0fc441fa7..83a653dcbc 100644 --- a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java @@ -97,6 +97,7 @@ import org.springframework.ai.chat.observation.DefaultChatModelObservationConvention; import org.springframework.ai.chat.prompt.ChatOptions; import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.content.ContentPart; import org.springframework.ai.content.Media; import org.springframework.ai.model.tool.ToolCallingManager; import org.springframework.ai.observation.conventions.AiProvider; @@ -513,92 +514,29 @@ ChatCompletionCreateParams createRequest(Prompt prompt, boolean stream) { // Handle simple text content for user and system messages ChatCompletionUserMessageParam.Builder builder = ChatCompletionUserMessageParam.builder(); - if (message instanceof UserMessage userMessage - && !CollectionUtils.isEmpty(userMessage.getMedia())) { - // Handle media content (images, audio, files) + if (message instanceof UserMessage userMessage && (userMessage.hasInterleavedContent() + || !CollectionUtils.isEmpty(userMessage.getMedia()))) { + // Handle media content (images, audio, files). Walking the + // content + // parts preserves the caller's text/media ordering; for a message + // built in the flat form the parts are the text followed by the + // media, so the resulting parts array is unchanged. List parts = new ArrayList<>(); - String messageText = message.getText(); - if (messageText != null && !messageText.isEmpty()) { - parts.add(ChatCompletionContentPart - .ofText(ChatCompletionContentPartText.builder().text(messageText).build())); - } - - // Add media content parts - userMessage.getMedia().forEach(media -> { - String mimeType = media.getMimeType().toString(); - if (mimeType.startsWith("image/")) { - if (media.getData() instanceof java.net.URI uri) { - parts.add(ChatCompletionContentPart - .ofImageUrl(ChatCompletionContentPartImage.builder() - .imageUrl(ChatCompletionContentPartImage.ImageUrl.builder() - .url(uri.toString()) - .build()) - .build())); - } - else if (media.getData() instanceof String text) { - // The org.springframework.ai.content.Media object - // should store the URL as a java.net.URI but it - // transforms it to String somewhere along the way, - // for example in its Builder class. So, we accept - // String as well here for image URLs. + for (ContentPart contentPart : userMessage.getContentParts()) { + if (contentPart instanceof ContentPart.TextPart textPart) { + if (!textPart.text().isEmpty()) { parts.add(ChatCompletionContentPart - .ofImageUrl(ChatCompletionContentPartImage.builder() - .imageUrl( - ChatCompletionContentPartImage.ImageUrl.builder().url(text).build()) - .build())); - } - else if (media.getData() instanceof byte[] bytes) { - // Assume the bytes are an image. So, convert the - // bytes to a base64 encoded - ChatCompletionContentPartImage.ImageUrl.Builder imageUrlBuilder = ChatCompletionContentPartImage.ImageUrl - .builder(); - - imageUrlBuilder.url("data:" + mimeType + ";base64," - + Base64.getEncoder().encodeToString(bytes)); - parts.add(ChatCompletionContentPart - .ofImageUrl(ChatCompletionContentPartImage.builder() - .imageUrl(imageUrlBuilder.build()) - .build())); - } - else { - if (logger.isInfoEnabled()) { - logger.info("Could not process image media with data of type: " - + media.getData().getClass().getSimpleName() - + ". Only java.net.URI is supported for image URLs."); - } + .ofText(ChatCompletionContentPartText.builder().text(textPart.text()).build())); } } - else if (mimeType.startsWith("audio/")) { - parts.add(ChatCompletionContentPart - .ofInputAudio(ChatCompletionContentPartInputAudio.builder() - .inputAudio(ChatCompletionContentPartInputAudio.builder() - .inputAudio(ChatCompletionContentPartInputAudio.InputAudio.builder() - .data(fromAudioData(media.getData())) - .format(mimeType.contains("mp3") - ? ChatCompletionContentPartInputAudio.InputAudio.Format.MP3 - : ChatCompletionContentPartInputAudio.InputAudio.Format.WAV) - .build()) - .build() - .inputAudio()) - .build())); - } - else if ("application/pdf".equals(mimeType)) { - parts.add(ChatCompletionContentPart.ofFile(ChatCompletionContentPart.File.builder() - .file(ChatCompletionContentPart.File.FileObject.builder() - .filename(media.getName()) - .fileData(fromMediaData(media.getMimeType(), media.getData())) - .build()) - .build())); - } - else { - // Assume it's a file or other media type represented as a - // data URL - parts.add(ChatCompletionContentPart.ofText(ChatCompletionContentPartText.builder() - .text(fromMediaData(media.getMimeType(), media.getData())) - .build())); + else if (contentPart instanceof ContentPart.MediaPart mediaPart) { + ChatCompletionContentPart mediaContentPart = toContentPart(mediaPart.media()); + if (mediaContentPart != null) { + parts.add(mediaContentPart); + } } - }); + } builder.contentOfArrayOfContentParts(parts); } else { @@ -952,6 +890,79 @@ public static ChatCompletionToolChoiceOption parseToolChoice(JsonNode node) { } } + /** + * Maps a single {@link Media} to the OpenAI content part that carries it. + * @param media the media to map + * @return the content part, or null when the media cannot be represented (an image + * whose data is of an unsupported type), in which case it is left out of the request + */ + private @Nullable ChatCompletionContentPart toContentPart(Media media) { + String mimeType = media.getMimeType().toString(); + if (mimeType.startsWith("image/")) { + if (media.getData() instanceof java.net.URI uri) { + return ChatCompletionContentPart.ofImageUrl(ChatCompletionContentPartImage.builder() + .imageUrl(ChatCompletionContentPartImage.ImageUrl.builder().url(uri.toString()).build()) + .build()); + } + else if (media.getData() instanceof String text) { + // The org.springframework.ai.content.Media object should store the URL as + // a + // java.net.URI but it transforms it to String somewhere along the way, + // for + // example in its Builder class. So, we accept String as well here for + // image + // URLs. + return ChatCompletionContentPart.ofImageUrl(ChatCompletionContentPartImage.builder() + .imageUrl(ChatCompletionContentPartImage.ImageUrl.builder().url(text).build()) + .build()); + } + else if (media.getData() instanceof byte[] bytes) { + // Assume the bytes are an image. So, convert the bytes to a base64 + // encoded + ChatCompletionContentPartImage.ImageUrl.Builder imageUrlBuilder = ChatCompletionContentPartImage.ImageUrl + .builder(); + + imageUrlBuilder.url("data:" + mimeType + ";base64," + Base64.getEncoder().encodeToString(bytes)); + return ChatCompletionContentPart + .ofImageUrl(ChatCompletionContentPartImage.builder().imageUrl(imageUrlBuilder.build()).build()); + } + else { + if (logger.isInfoEnabled()) { + logger.info("Could not process image media with data of type: " + + media.getData().getClass().getSimpleName() + + ". Only java.net.URI is supported for image URLs."); + } + return null; + } + } + else if (mimeType.startsWith("audio/")) { + return ChatCompletionContentPart.ofInputAudio(ChatCompletionContentPartInputAudio.builder() + .inputAudio(ChatCompletionContentPartInputAudio.builder() + .inputAudio(ChatCompletionContentPartInputAudio.InputAudio.builder() + .data(fromAudioData(media.getData())) + .format(mimeType.contains("mp3") ? ChatCompletionContentPartInputAudio.InputAudio.Format.MP3 + : ChatCompletionContentPartInputAudio.InputAudio.Format.WAV) + .build()) + .build() + .inputAudio()) + .build()); + } + else if ("application/pdf".equals(mimeType)) { + return ChatCompletionContentPart.ofFile(ChatCompletionContentPart.File.builder() + .file(ChatCompletionContentPart.File.FileObject.builder() + .filename(media.getName()) + .fileData(fromMediaData(media.getMimeType(), media.getData())) + .build()) + .build()); + } + else { + // Assume it's a file or other media type represented as a data URL + return ChatCompletionContentPart.ofText(ChatCompletionContentPartText.builder() + .text(fromMediaData(media.getMimeType(), media.getData())) + .build()); + } + } + private String fromAudioData(Object audioData) { if (audioData instanceof byte[] bytes) { return Base64.getEncoder().encodeToString(bytes); diff --git a/spring-ai-commons/src/main/java/org/springframework/ai/content/ContentPart.java b/spring-ai-commons/src/main/java/org/springframework/ai/content/ContentPart.java new file mode 100644 index 0000000000..1a8015003c --- /dev/null +++ b/spring-ai-commons/src/main/java/org/springframework/ai/content/ContentPart.java @@ -0,0 +1,93 @@ +/* + * Copyright 2023-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.content; + +import org.springframework.util.Assert; + +/** + * A single ordered element of a message's content, allowing text and media to be + * interleaved in a caller-defined sequence. + *

+ * A message that carries text alongside media in two separate collections can only ever + * be serialized as "all text, then all media". Some multimodal prompts depend on a finer + * ordering: a per-page document prompt, for instance, needs each page's image to directly + * follow that page's own text so the model associates the two. Expressing the content as + * an ordered {@code List} makes such prompts representable. + *

+ * Providers whose wire format is itself an ordered list of parts (Google GenAI, + * Anthropic, OpenAI, Amazon Bedrock, Mistral AI) map content parts one-to-one. Providers + * with a flat wire format receive the content flattened back to text-plus-media, which is + * the most their API can express. + * + * @author Dmitrii Erokhin + * @since 2.0.1 + * @see TextPart + * @see MediaPart + */ +public sealed interface ContentPart { + + /** + * Creates a text content part. + * @param text the text of this part; must not be null, but may be empty + * @return a new text part + */ + static ContentPart text(String text) { + return new TextPart(text); + } + + /** + * Creates a media content part. + * @param media the media of this part; must not be null + * @return a new media part + */ + static ContentPart media(Media media) { + return new MediaPart(media); + } + + /** + * A text fragment within a message's content. + *

+ * Empty and whitespace-only text is permitted, so that content parts derived from a + * message built with blank text round-trip faithfully. Providers should skip blank + * text parts when serializing, since several model APIs reject empty text blocks. + * + * @param text the text of this part + * @since 2.0.1 + */ + record TextPart(String text) implements ContentPart { + + public TextPart { + Assert.notNull(text, "text cannot be null"); + } + + } + + /** + * A media fragment within a message's content. + * + * @param media the media of this part + * @since 2.0.1 + */ + record MediaPart(Media media) implements ContentPart { + + public MediaPart { + Assert.notNull(media, "media cannot be null"); + } + + } + +} diff --git a/spring-ai-commons/src/test/java/org/springframework/ai/content/ContentPartTests.java b/spring-ai-commons/src/test/java/org/springframework/ai/content/ContentPartTests.java new file mode 100644 index 0000000000..bc50479e60 --- /dev/null +++ b/spring-ai-commons/src/test/java/org/springframework/ai/content/ContentPartTests.java @@ -0,0 +1,96 @@ +/* + * Copyright 2023-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.content; + +import java.net.URI; + +import org.junit.jupiter.api.Test; + +import org.springframework.util.MimeTypeUtils; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Unit tests for {@link ContentPart}. + * + * @author Dmitrii Erokhin + */ +class ContentPartTests { + + private static Media testMedia() { + return new Media(MimeTypeUtils.IMAGE_PNG, URI.create("https://example.com/image.png")); + } + + @Test + void textFactoryCreatesTextPart() { + ContentPart part = ContentPart.text("Hello, world!"); + + assertThat(part).isInstanceOf(ContentPart.TextPart.class); + assertThat(((ContentPart.TextPart) part).text()).isEqualTo("Hello, world!"); + } + + @Test + void mediaFactoryCreatesMediaPart() { + Media media = testMedia(); + + ContentPart part = ContentPart.media(media); + + assertThat(part).isInstanceOf(ContentPart.MediaPart.class); + assertThat(((ContentPart.MediaPart) part).media()).isSameAs(media); + } + + @Test + void textPartRejectsNullText() { + assertThatThrownBy(() -> new ContentPart.TextPart(null)).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("text cannot be null"); + } + + @Test + void mediaPartRejectsNullMedia() { + assertThatThrownBy(() -> new ContentPart.MediaPart(null)).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("media cannot be null"); + } + + @Test + void textPartAllowsEmptyAndBlankText() { + // Blank text parts must be representable so that content derived from a message + // built with blank text round-trips unchanged. Providers skip them when + // serializing. + assertThat(new ContentPart.TextPart("").text()).isEmpty(); + assertThat(new ContentPart.TextPart(" \t\n ").text()).isEqualTo(" \t\n "); + } + + @Test + void textPartsWithEqualTextAreEqual() { + assertThat(ContentPart.text("same")).isEqualTo(ContentPart.text("same")) + .hasSameHashCodeAs(ContentPart.text("same")); + assertThat(ContentPart.text("one")).isNotEqualTo(ContentPart.text("two")); + } + + @Test + void mediaPartEqualityFollowsMediaIdentity() { + Media media = testMedia(); + + // Media does not override equals/hashCode, so MediaPart equality is necessarily + // identity-based on the wrapped Media. Pinned here so the limitation is visible: + // value-based equality requires Media#equals first. + assertThat(ContentPart.media(media)).isEqualTo(ContentPart.media(media)); + assertThat(ContentPart.media(testMedia())).isNotEqualTo(ContentPart.media(testMedia())); + } + +} diff --git a/spring-ai-model/src/main/java/org/springframework/ai/aot/SpringAiCoreRuntimeHints.java b/spring-ai-model/src/main/java/org/springframework/ai/aot/SpringAiCoreRuntimeHints.java index 2bc0dfc812..8f037b66e5 100644 --- a/spring-ai-model/src/main/java/org/springframework/ai/aot/SpringAiCoreRuntimeHints.java +++ b/spring-ai-model/src/main/java/org/springframework/ai/aot/SpringAiCoreRuntimeHints.java @@ -28,6 +28,7 @@ import org.springframework.ai.chat.messages.ToolResponseMessage; import org.springframework.ai.chat.messages.UserMessage; import org.springframework.ai.content.Content; +import org.springframework.ai.content.ContentPart; import org.springframework.ai.content.MediaContent; import org.springframework.ai.tool.ToolCallback; import org.springframework.ai.tool.definition.ToolDefinition; @@ -43,7 +44,7 @@ public void registerHints(RuntimeHints hints, @Nullable ClassLoader classLoader) var chatTypes = Set.of(AbstractMessage.class, AssistantMessage.class, ToolResponseMessage.class, Message.class, ToolCallback.class, ToolDefinition.class, AssistantMessage.ToolCall.class, MessageType.class, - UserMessage.class, SystemMessage.class, Content.class, MediaContent.class); + UserMessage.class, SystemMessage.class, Content.class, MediaContent.class, ContentPart.class); var memberCategories = MemberCategory.values(); diff --git a/spring-ai-model/src/main/java/org/springframework/ai/chat/messages/UserMessage.java b/spring-ai-model/src/main/java/org/springframework/ai/chat/messages/UserMessage.java index 005df3d320..036e9c9772 100644 --- a/spring-ai-model/src/main/java/org/springframework/ai/chat/messages/UserMessage.java +++ b/spring-ai-model/src/main/java/org/springframework/ai/chat/messages/UserMessage.java @@ -25,6 +25,7 @@ import org.jspecify.annotations.Nullable; +import org.springframework.ai.content.ContentPart; import org.springframework.ai.content.Media; import org.springframework.ai.content.MediaContent; import org.springframework.core.io.Resource; @@ -35,11 +36,33 @@ * A message of the type 'user' passed as input Messages with the user role are from the * end-user or developer. They represent questions, prompts, or any input that you want * the generative to respond to. + *

+ * The content of a user message can be expressed in two equivalent ways. The flat form + * pairs a single text with a collection of media, which every provider serializes as text + * followed by all media. The ordered form, set through + * {@link Builder#contentParts(List)}, is a {@link ContentPart} list that interleaves text + * and media in a caller-defined sequence — required by prompts where each media item must + * directly follow its own text. + *

+ * The two forms are always both available: {@link #getText()} and {@link #getMedia()} are + * projections of {@link #getContentParts()} and vice versa, so a message built either way + * reads correctly through either accessor and providers that cannot express ordering + * degrade to text-then-media automatically. */ public class UserMessage extends AbstractMessage implements MediaContent { + /** + * Separator used to join the text of multiple text parts into the flat + * {@link #getText()} projection. Deliberately a literal newline rather than + * {@code System.lineSeparator()}: the projection is persisted and participates in + * {@link #equals(Object)}, so it must not vary by platform. + */ + private static final String TEXT_PART_SEPARATOR = "\n"; + protected final List media; + private final List contentParts; + public UserMessage(@Nullable String textContent) { this(textContent, new ArrayList<>(), Map.of()); } @@ -48,33 +71,147 @@ private UserMessage(@Nullable String textContent, Collection media, Map(media); + this.media = List.copyOf(media); + this.contentParts = deriveContentParts(textContent, media); + } + + private UserMessage(List contentParts, Map metadata) { + super(MessageType.USER, joinText(contentParts), metadata); + this.contentParts = List.copyOf(contentParts); + this.media = deriveMedia(contentParts); } public UserMessage(Resource resource) { this(MessageUtils.readResource(resource)); } + /** + * Projects the flat text-plus-media form onto an ordered content part list. + * Derivation keys on {@code textContent != null} rather than on the text being + * non-blank, so that a message carrying empty or whitespace-only text round-trips + * unchanged through {@link #copy()}. + */ + private static List deriveContentParts(@Nullable String textContent, Collection media) { + List parts = new ArrayList<>(media.size() + 1); + if (textContent != null) { + parts.add(new ContentPart.TextPart(textContent)); + } + for (Media mediaItem : media) { + parts.add(new ContentPart.MediaPart(mediaItem)); + } + return List.copyOf(parts); + } + + /** + * Projects an ordered content part list onto the flat text form. Media parts + * contribute nothing: components that treat user text as a query (retrieval + * augmentation, for instance) would otherwise embed a media marker into that query. + */ + private static String joinText(List contentParts) { + Assert.notNull(contentParts, "contentParts cannot be null"); + StringBuilder text = new StringBuilder(); + boolean firstTextPart = true; + for (ContentPart part : contentParts) { + if (part instanceof ContentPart.TextPart textPart) { + if (!firstTextPart) { + text.append(TEXT_PART_SEPARATOR); + } + text.append(textPart.text()); + firstTextPart = false; + } + } + return text.toString(); + } + + /** + * Projects an ordered content part list onto the flat media form, preserving the + * order in which the media parts appear. + */ + private static List deriveMedia(List contentParts) { + List media = new ArrayList<>(); + for (ContentPart part : contentParts) { + if (part instanceof ContentPart.MediaPart mediaPart) { + media.add(mediaPart.media()); + } + } + return List.copyOf(media); + } + @Override public String toString() { return "UserMessage{" + "content='" + getText() + '\'' + ", metadata=" + this.metadata + ", messageType=" + this.messageType + '}'; } + /** + * Returns the media of this message, in order. The returned list is unmodifiable: + * mutating it would desynchronize it from {@link #getContentParts()}. + */ @Override public List getMedia() { return this.media; } + /** + * Returns the ordered content of this message as text and media parts. + *

+ * Never empty for a message that carries any text or media — for a message built in + * the flat form this returns the derived {@code [text, media…]} sequence — so + * providers need only one code path. The parts may include a text part whose text is + * empty or whitespace-only, which providers should skip when serializing since + * several model APIs reject empty text blocks. + * @return the unmodifiable, ordered content parts of this message + * @since 2.0.1 + */ + public List getContentParts() { + return this.contentParts; + } + + /** + * Whether this message's content needs the ordered form to be expressed faithfully, + * which is the case when the parts are anything other than a single text followed by + * media — for instance media followed by more text, or several separate texts. + *

+ * Providers whose wire format offers a flat text-plus-media shortcut can keep taking + * it while this returns {@code false}, because flattening then loses nothing: the + * parts are exactly what {@link #getText()} and {@link #getMedia()} already convey. + * When it returns {@code true}, only {@link #getContentParts()} carries the full + * content. + * @return true if flattening this message's content to text-plus-media would lose + * information + * @since 2.0.1 + */ + public boolean hasInterleavedContent() { + boolean textSeen = false; + boolean mediaSeen = false; + for (ContentPart part : this.contentParts) { + if (part instanceof ContentPart.TextPart) { + if (textSeen || mediaSeen) { + return true; + } + textSeen = true; + } + else { + mediaSeen = true; + } + } + return false; + } + public UserMessage copy() { return mutate().build(); } public Builder mutate() { - Builder builder = new Builder().media(List.copyOf(getMedia())).metadata(Map.copyOf(getMetadata())); + Builder builder = new Builder().metadata(Map.copyOf(getMetadata())); + // Seed the flat slots first and the ordered content last: contentParts wins, but + // the flat slots stay populated so that a caller who overwrites the text with + // text(...) still keeps this message's media. + builder.media(List.copyOf(getMedia())); if (this.textContent != null) { builder.text(this.textContent); } + builder.contentParts(this.contentParts); return builder; } @@ -90,25 +227,105 @@ public static final class Builder { private List media = new ArrayList<>(); + private @Nullable List contentParts; + private Map metadata = new HashMap<>(); + /** + * Sets the text of the message, discarding any content parts previously set on + * this builder. The resulting message carries the given text followed by whatever + * media is set, losing any interleaving — which is what lets components that + * rewrite user text keep working against ordered content. + */ public Builder text(String textContent) { this.textContent = textContent; + this.contentParts = null; return this; } + /** + * Sets the text of the message from a resource, discarding any content parts + * previously set on this builder. + */ public Builder text(Resource resource) { this.resource = resource; + this.contentParts = null; return this; } + /** + * Sets the media of the message, discarding any content parts previously set on + * this builder. + */ public Builder media(List media) { this.media = media; + this.contentParts = null; return this; } + /** + * Sets the media of the message, discarding any content parts previously set on + * this builder. + */ public Builder media(Media... media) { this.media = Arrays.asList(media); + this.contentParts = null; + return this; + } + + /** + * Sets the ordered content of the message as text and media parts, taking + * precedence over any text and media previously set on this builder. + * @param contentParts the ordered content parts; must not be null or contain null + * elements + * @return this builder + * @since 2.0.1 + */ + public Builder contentParts(List contentParts) { + Assert.notNull(contentParts, "contentParts cannot be null"); + Assert.noNullElements(contentParts, "contentParts cannot have null elements"); + this.contentParts = List.copyOf(contentParts); + return this; + } + + /** + * Sets the ordered content of the message as text and media parts, taking + * precedence over any text and media previously set on this builder. + * @param contentParts the ordered content parts; must not be null or contain null + * elements + * @return this builder + * @since 2.0.1 + */ + public Builder contentParts(ContentPart... contentParts) { + Assert.notNull(contentParts, "contentParts cannot be null"); + return contentParts(Arrays.asList(contentParts)); + } + + /** + * Appends text to the end of the message's content, preserving any ordering + * already established. + *

+ * Where {@link #text(String)} replaces the content and flattens it, this adds a + * trailing text part to ordered content, or concatenates onto the existing text + * of flat content. Flat content is concatenated verbatim, with no separator + * inserted; for ordered content the appended text becomes its own part, so the + * flat {@link #getText()} projection separates it from the preceding text with a + * newline. Null and blank text are ignored. + * @param text the text to append + * @return this builder + * @since 2.0.1 + */ + public Builder appendText(@Nullable String text) { + if (!StringUtils.hasText(text)) { + return this; + } + if (this.contentParts != null) { + List appended = new ArrayList<>(this.contentParts); + appended.add(new ContentPart.TextPart(text)); + this.contentParts = List.copyOf(appended); + return this; + } + this.textContent = (this.textContent != null) ? this.textContent + text : text; return this; } @@ -118,6 +335,9 @@ public Builder metadata(Map metadata) { } public UserMessage build() { + if (this.contentParts != null) { + return new UserMessage(this.contentParts, this.metadata); + } if (StringUtils.hasText(this.textContent) && this.resource != null) { throw new IllegalArgumentException("textContent and resource cannot be set at the same time"); } diff --git a/spring-ai-model/src/test/java/org/springframework/ai/chat/messages/UserMessageTests.java b/spring-ai-model/src/test/java/org/springframework/ai/chat/messages/UserMessageTests.java index 41516849b5..9ca651b39a 100644 --- a/spring-ai-model/src/test/java/org/springframework/ai/chat/messages/UserMessageTests.java +++ b/spring-ai-model/src/test/java/org/springframework/ai/chat/messages/UserMessageTests.java @@ -16,10 +16,15 @@ package org.springframework.ai.chat.messages; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; +import org.springframework.ai.content.ContentPart; import org.springframework.ai.content.Media; import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.Resource; @@ -263,4 +268,271 @@ void userMessageToStringWithMedia() { assertThat(toString).contains("UserMessage").contains(text).contains("media"); } + private static Media imageMedia() { + return new Media(MimeTypeUtils.IMAGE_PNG, URI.create("https://example.com/image.png")); + } + + @Test + void userMessageFromContentPartsPreservesOrder() { + Media media1 = imageMedia(); + Media media2 = imageMedia(); + UserMessage message = UserMessage.builder() + .contentParts(ContentPart.text("--- p1 ---"), ContentPart.media(media1), ContentPart.text("--- p2 ---"), + ContentPart.media(media2)) + .build(); + + assertThat(message.getContentParts()).containsExactly(ContentPart.text("--- p1 ---"), ContentPart.media(media1), + ContentPart.text("--- p2 ---"), ContentPart.media(media2)); + assertThat(message.getText()).isEqualTo("--- p1 ---\n--- p2 ---"); + assertThat(message.getMedia()).containsExactly(media1, media2); + } + + @Test + void userMessageFromContentPartsJoinsTextWithLiteralNewline() { + UserMessage message = UserMessage.builder() + .contentParts(ContentPart.text("first"), ContentPart.text("second")) + .build(); + + // A literal "\n", never System.lineSeparator(): the projection is persisted and + // participates in equals, so it must not vary by platform. + assertThat(message.getText()).isEqualTo("first\nsecond"); + } + + @Test + void userMessageFromContentPartsWithMediaOnly() { + Media media = imageMedia(); + UserMessage message = UserMessage.builder().contentParts(ContentPart.media(media)).build(); + + assertThat(message.getText()).isEmpty(); + assertThat(message.getMedia()).containsExactly(media); + assertThat(message.getContentParts()).hasSize(1); + } + + @Test + void userMessageFromEmptyContentParts() { + UserMessage message = UserMessage.builder().contentParts(List.of()).build(); + + assertThat(message.getText()).isEmpty(); + assertThat(message.getMedia()).isEmpty(); + assertThat(message.getContentParts()).isEmpty(); + } + + @Test + void userMessageTextProjectionOmitsMedia() { + // Components that treat user text as a retrieval query read getText() directly, + // so + // no media marker may leak into it. + Media media = imageMedia(); + UserMessage message = UserMessage.builder() + .contentParts(ContentPart.text("what is this?"), ContentPart.media(media)) + .build(); + + assertThat(message.getText()).isEqualTo("what is this?"); + assertThat(message.getText()).doesNotContain(media.getName()); + } + + @Test + void userMessageDerivesContentPartsFromTextAndMedia() { + Media media1 = imageMedia(); + Media media2 = imageMedia(); + UserMessage message = UserMessage.builder().text("Hello").media(media1, media2).build(); + + assertThat(message.getContentParts()).containsExactly(ContentPart.text("Hello"), ContentPart.media(media1), + ContentPart.media(media2)); + } + + @Test + void userMessageDerivesContentPartsFromBlankText() { + // Derivation keys on text != null, not on the text being non-blank, so blank text + // round-trips through copy(). + assertThat(new UserMessage("").getContentParts()).containsExactly(ContentPart.text("")); + assertThat(new UserMessage(" \t\n ").getContentParts()).containsExactly(ContentPart.text(" \t\n ")); + } + + @Test + void userMessageBuilderTextDiscardsContentParts() { + Media media = imageMedia(); + UserMessage message = UserMessage.builder() + .contentParts(ContentPart.text("ordered"), ContentPart.media(media)) + .text("flat") + .build(); + + assertThat(message.getText()).isEqualTo("flat"); + assertThat(message.getContentParts()).containsExactly(ContentPart.text("flat")); + assertThat(message.getMedia()).isEmpty(); + } + + @Test + void userMessageBuilderMediaDiscardsContentParts() { + Media ordered = imageMedia(); + Media flat = imageMedia(); + UserMessage message = UserMessage.builder() + .contentParts(ContentPart.text("ordered"), ContentPart.media(ordered)) + .text("flat") + .media(flat) + .build(); + + assertThat(message.getContentParts()).containsExactly(ContentPart.text("flat"), ContentPart.media(flat)); + } + + @Test + void userMessageBuilderContentPartsOverridesText() { + UserMessage message = UserMessage.builder().text("discarded").contentParts(ContentPart.text("ordered")).build(); + + assertThat(message.getText()).isEqualTo("ordered"); + assertThat(message.getContentParts()).containsExactly(ContentPart.text("ordered")); + } + + @Test + void userMessageBuilderStillRejectsTextAndResourceTogether() { + assertThatThrownBy( + () -> UserMessage.builder().text("some text").text(new ClassPathResource("prompt-user.txt")).build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("textContent and resource cannot be set at the same time"); + } + + @Test + void userMessageCopyPreservesContentParts() { + Media media = imageMedia(); + UserMessage original = UserMessage.builder() + .contentParts(ContentPart.text("a"), ContentPart.media(media), ContentPart.text("b")) + .build(); + + UserMessage copy = original.copy(); + + assertThat(copy).isNotSameAs(original); + assertThat(copy.getContentParts()).isEqualTo(original.getContentParts()); + assertThat(copy.getText()).isEqualTo("a\nb"); + } + + @Test + void userMessageMutateWithTextFlattensWithoutStaleText() { + Media media = imageMedia(); + UserMessage original = UserMessage.builder() + .contentParts(ContentPart.text("old"), ContentPart.media(media)) + .build(); + + UserMessage mutated = original.mutate().text("replaced").build(); + + // The whole point of eager invalidation: no part may still carry the old text, or + // a + // parts-aware provider would silently send a prompt the caller already replaced. + assertThat(mutated.getText()).isEqualTo("replaced"); + assertThat(mutated.getContentParts()).doesNotContain(ContentPart.text("old")); + assertThat(mutated.getContentParts()).containsExactly(ContentPart.text("replaced"), ContentPart.media(media)); + assertThat(mutated.getMedia()).containsExactly(media); + } + + @Test + void userMessageMutateAppendTextPreservesContentParts() { + Media media = imageMedia(); + UserMessage original = UserMessage.builder() + .contentParts(ContentPart.text("body"), ContentPart.media(media)) + .build(); + + UserMessage appended = original.mutate().appendText("FORMAT").build(); + + assertThat(appended.getContentParts()).containsExactly(ContentPart.text("body"), ContentPart.media(media), + ContentPart.text("FORMAT")); + assertThat(appended.getMedia()).containsExactly(media); + } + + @Test + void userMessageAppendTextConcatenatesFlatText() { + UserMessage message = UserMessage.builder().text("body").appendText("\nFORMAT").build(); + + assertThat(message.getText()).isEqualTo("body\nFORMAT"); + assertThat(message.getContentParts()).containsExactly(ContentPart.text("body\nFORMAT")); + } + + @Test + void userMessageAppendTextIgnoresNullAndBlank() { + UserMessage message = UserMessage.builder().text("body").appendText(null).appendText(" ").build(); + + assertThat(message.getText()).isEqualTo("body"); + } + + @Test + void userMessageBuilderRejectsNullContentParts() { + assertThatThrownBy(() -> UserMessage.builder().contentParts((List) null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("contentParts cannot be null"); + + assertThatThrownBy(() -> UserMessage.builder().contentParts(Arrays.asList(ContentPart.text("a"), null))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("contentParts cannot have null elements"); + } + + @Test + void userMessageContentPartsAndMediaAreUnmodifiable() { + Media media = imageMedia(); + UserMessage message = UserMessage.builder() + .contentParts(ContentPart.text("a"), ContentPart.media(media)) + .build(); + + assertThatThrownBy(() -> message.getContentParts().add(ContentPart.text("b"))) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> message.getMedia().add(imageMedia())) + .isInstanceOf(UnsupportedOperationException.class); + } + + @Test + void userMessageContentPartsIndependentOfCallerList() { + List parts = new ArrayList<>(); + parts.add(ContentPart.text("a")); + UserMessage message = UserMessage.builder().contentParts(parts).build(); + + parts.add(ContentPart.text("b")); + + assertThat(message.getContentParts()).containsExactly(ContentPart.text("a")); + assertThat(message.getText()).isEqualTo("a"); + } + + @Test + void userMessageHasInterleavedContentOnlyWhenFlatteningWouldLose() { + Media media = imageMedia(); + + // Flat-equivalent shapes: a single leading text, then media. Providers with a + // flat + // fast path may keep using it for these. + assertThat(new UserMessage("just text").hasInterleavedContent()).isFalse(); + assertThat(UserMessage.builder().text("text").media(media).build().hasInterleavedContent()).isFalse(); + assertThat(UserMessage.builder() + .contentParts(ContentPart.text("text"), ContentPart.media(media)) + .build() + .hasInterleavedContent()).isFalse(); + assertThat(UserMessage.builder().contentParts(ContentPart.media(media)).build().hasInterleavedContent()) + .isFalse(); + assertThat(UserMessage.builder().contentParts(List.of()).build().hasInterleavedContent()).isFalse(); + + // Shapes that only the ordered form can express: text after media, or several + // texts + // (whose separate blocks a provider concatenates without the projection's + // newline). + assertThat(UserMessage.builder() + .contentParts(ContentPart.media(media), ContentPart.text("after")) + .build() + .hasInterleavedContent()).isTrue(); + assertThat(UserMessage.builder() + .contentParts(ContentPart.text("one"), ContentPart.text("two")) + .build() + .hasInterleavedContent()).isTrue(); + } + + @Test + void userMessageEqualityIgnoresContentPartOrdering() { + Media media = imageMedia(); + UserMessage textFirst = UserMessage.builder() + .contentParts(ContentPart.text("a"), ContentPart.media(media)) + .build(); + UserMessage mediaFirst = UserMessage.builder() + .contentParts(ContentPart.media(media), ContentPart.text("a")) + .build(); + + // Pinning existing semantics, not endorsing them: UserMessage does not override + // equals/hashCode, so neither media nor part ordering participates in equality. + // Fixing that requires value-based equals on Media first. + assertThat(textFirst).isEqualTo(mediaFirst); + } + }