From 0acb2150400cc9626dd33898c525d2d9e7c5c772 Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Fri, 3 Jul 2026 12:49:03 +0200 Subject: [PATCH 01/17] Add verbose flags to build and test scripts --- bin/build.sh | 19 ++++++++++++++++--- bin/test.sh | 37 ++++++++++++++++++++++++++++--------- 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/bin/build.sh b/bin/build.sh index 27bcd3c..845add7 100755 --- a/bin/build.sh +++ b/bin/build.sh @@ -9,16 +9,29 @@ if [[ "${TRACE-0}" == "1" ]]; then fi if [[ "${1-}" =~ ^-*h(elp)?$ ]]; then - echo 'Usage: ./build.sh' + echo 'Usage: ./build.sh [--verbose|-v]' + echo + echo 'Build the SHLLM library (Xcode/Metal aware).' + echo + echo 'Options:' + echo ' --verbose, -v Bypass xcbeautify and show raw xcodebuild output.' exit fi +verbose=false +for arg in "$@"; do + case "$arg" in + --verbose|-v) verbose=true ;; + *) echo "Unknown argument: $arg" >&2; exit 1 ;; + esac +done + DIR=$(dirname "$0") pushd "$DIR/.." &>/dev/null beautify="" -if command -v xcbeautify &>/dev/null; then - beautify="| xcbeautify" +if ! $verbose && command -v xcbeautify &>/dev/null; then + beautify="2>&1 | xcbeautify" fi eval "exec xcodebuild \ diff --git a/bin/test.sh b/bin/test.sh index 354affd..16b870c 100755 --- a/bin/test.sh +++ b/bin/test.sh @@ -3,9 +3,28 @@ set -eo pipefail cd "$(dirname $0)/.." +verbose=false testSpecifiers=() for arg in "$@"; do - testSpecifiers+=("-only-testing:$arg") + case "$arg" in + --verbose|-v) verbose=true ;; + -h|--help) + echo 'Usage: ./test.sh [--verbose|-v] [TEST_SPECIFIER ...]' + echo + echo 'Run SHLLM tests (Xcode/Metal aware).' + echo + echo 'Options:' + echo ' --verbose, -v Bypass xcbeautify and show raw xcodebuild output.' + echo + echo 'Test specifiers are passed to xcodebuild as -only-testing: arguments.' + echo 'Examples:' + echo ' ./test.sh' + echo ' ./test.sh SHLLMTests/ResponseParserTests' + echo ' ./test.sh -v SHLLMTests/Gemma4_E2BTests/canStreamResult()' + exit + ;; + *) testSpecifiers+=("-only-testing:$arg") ;; + esac done xcodeVersion=$(xcodebuild -version | sed -n 's/Xcode \([0-9]*\).*/\1/p') @@ -15,25 +34,25 @@ if [ "$xcodeVersion" -ge 26 ]; then echo "⬇️ Downloading Metal toolchain..." eval "exec xcodebuild \ - -downloadComponent metalToolchain - -exportPath /tmp/metalToolchainDownload/ ${beautify}" + -downloadComponent metalToolchain \ + -exportPath /tmp/metalToolchainDownload/" echo "🧰 Installing Metal toolchain..." - eval "exec xcodebuild - -importComponent metalToolchain - -importPath /tmp/metalToolchainDownload/*.exportedBundle ${beautify}" + eval "exec xcodebuild \ + -importComponent metalToolchain \ + -importPath /tmp/metalToolchainDownload/*.exportedBundle" fi fi signingFlags="CODE_SIGNING_ALLOWED=NO CODE_SIGNING_REQUIRED=NO" -if command -v xcbeautify &>/dev/null; then +if ! $verbose && command -v xcbeautify &>/dev/null; then xcodebuild \ -scheme SHLLM \ -destination 'platform=OS X' \ ${signingFlags} \ "${testSpecifiers[@]}" \ - test | xcbeautify + test 2>&1 | xcbeautify else xcodebuild \ -scheme SHLLM \ @@ -41,4 +60,4 @@ else ${signingFlags} \ "${testSpecifiers[@]}" \ test -fi +fi \ No newline at end of file From 70a4a47fe511fe91d603146e6bc990dfec5503e8 Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Fri, 3 Jul 2026 12:54:47 +0200 Subject: [PATCH 02/17] Add Gemma 4 model support --- Package.resolved | 14 +- Package.swift | 5 +- Sources/SHLLM/LLM.swift | 129 +++++++++ Sources/SHLLM/ResponseParser.swift | 181 ++++++++++++ Sources/SHLLM/SHLLM.swift | 4 + .../SHLLMTests/Gemma4ChannelParserTests.swift | 188 +++++++++++++ Tests/SHLLMTests/Models/Gemma4-12BTests.swift | 263 ++++++++++++++++++ Tests/SHLLMTests/Models/Gemma4-E2BTests.swift | 263 ++++++++++++++++++ Tests/SHLLMTests/Models/Gemma4-E4BTests.swift | 263 ++++++++++++++++++ bin/download.sh | 3 + 10 files changed, 1305 insertions(+), 8 deletions(-) create mode 100644 Tests/SHLLMTests/Gemma4ChannelParserTests.swift create mode 100644 Tests/SHLLMTests/Models/Gemma4-12BTests.swift create mode 100644 Tests/SHLLMTests/Models/Gemma4-E2BTests.swift create mode 100644 Tests/SHLLMTests/Models/Gemma4-E4BTests.swift diff --git a/Package.resolved b/Package.resolved index 4b0ade9..c2fb260 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,13 +1,13 @@ { - "originHash" : "0e1ab4a07bf85e219a0e02bbc89d073a5568bda7a85e0c2b2b385df9881f40ce", + "originHash" : "41ae51a69b4ebad5129ae3c32f2396d7aaee03fe871f45bac58696c688b8718f", "pins" : [ { "identity" : "mlx-swift", "kind" : "remoteSourceControl", "location" : "https://github.com/ml-explore/mlx-swift", "state" : { - "revision" : "61b9e011e09a62b489f6bd647958f1555bdf2896", - "version" : "0.31.3" + "revision" : "dc43e62d7055353c7f99fa071a4e71d29dfddc44", + "version" : "0.31.4" } }, { @@ -15,8 +15,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/shareup/mlx-swift-lm", "state" : { - "revision" : "8825bdd20fbcf982171972a661016817363ca495", - "version" : "0.0.14" + "revision" : "bd4b7434e6bdb588c7ef55706ff8904cb7fd4c57", + "version" : "0.0.15" } }, { @@ -60,8 +60,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/swiftlang/swift-syntax.git", "state" : { - "revision" : "0687f71944021d616d34d922343dcef086855920", - "version" : "600.0.1" + "revision" : "79e4b74a295b6eb74a8b585e3a39d29e70c1dbd1", + "version" : "603.0.2" } }, { diff --git a/Package.swift b/Package.swift index d9ee721..ed22405 100644 --- a/Package.swift +++ b/Package.swift @@ -17,7 +17,7 @@ let package = Package( ), .package( url: "https://github.com/shareup/mlx-swift-lm", - from: "0.0.14" + from: "0.0.15" ), .package( url: "https://github.com/DePasqualeOrg/swift-tokenizers", @@ -49,6 +49,9 @@ let package = Package( // .copy("Resources/gemma-3-27b-it-qat-4bit"), // .copy("Resources/gemma-3-4b-it-qat-3bit"), // .copy("Resources/gemma-3-4b-it-qat-4bit"), +// .copy("Resources/gemma-4-e2b-it-4bit"), +// .copy("Resources/gemma-4-e4b-it-4bit"), +// .copy("Resources/gemma-4-12B-it-4bit"), // .copy("Resources/gpt-oss-20b-MLX-8bit"), // .copy("Resources/gpt-oss-20b-MXFP4-Q4"), // .copy("Resources/LFM2-8B-A1B-4bit"), diff --git a/Sources/SHLLM/LLM.swift b/Sources/SHLLM/LLM.swift index d6c1fcb..e20a378 100644 --- a/Sources/SHLLM/LLM.swift +++ b/Sources/SHLLM/LLM.swift @@ -625,6 +625,135 @@ extension LLM where Model == Gemma3 { } } +// MARK: - Gemma 4 Vision + +extension LLM where Model == Gemma4 { + /// **gemma-4-e2b** and **gemma-4-e4b** + /// + /// To enable thinking, set + /// `UserInput(additionalContext: ["enable_thinking": true])` + /// and add `<|think|>` to the **beginning** and **end** of + /// the system prompt. (The documentation says to just add it to + /// the beginning of the system prompt, but testing has shown it + /// to be more reliable when added to the beginning and end of the + /// system prompt. + /// + /// https://huggingface.co/google/gemma-4-12B-it#2-thinking-mode-configuration + public static func gemma4( + directory: URL, + input: UserInput, + tools: [any ToolProtocol] = [], + maxInputTokenCount: Int? = nil, + maxOutputTokenCount: Int? = nil + ) throws -> LLM { + try SHLLM.assertSupportedDevice + return .init( + directory: directory, + input: input, + tools: tools, + maxInputTokenCount: maxInputTokenCount, + maxOutputTokenCount: maxOutputTokenCount, + customConfiguration: { config in + var config = config + config.extraEOSTokens = [""] + return config + }, + generateParameters: generateParameters, + responseParser: gemma4Parser + ) + } + + // https://huggingface.co/google/gemma-4-E2B-it#best-practices + // + // 1. Sampling Parameters + // + // - temperature=1.0 + // - top_p=0.95 + // - top_k=64 + static var generateParameters: GenerateParameters { + GenerateParameters( + temperature: 1.0, + topP: 0.95, + topK: 64 + ) + } + + static var gemma4_E2B: URL { + get throws { + let dir = "gemma-4-e2b-it-4bit" + return try Bundle.shllm.directory(named: dir) + } + } + + static var gemma4_E4B: URL { + get throws { + let dir = "gemma-4-e4b-it-4bit" + return try Bundle.shllm.directory(named: dir) + } + } +} + +// MARK: - Gemma 4 Unified Vision + +extension LLM where Model == Gemma4Unified { + /// **gemma-4-12b** + /// + /// To enable thinking, set + /// `UserInput(additionalContext: ["enable_thinking": true])` + /// and add `<|think|>` to the **beginning** and **end** of + /// the system prompt. (The documentation says to just add it to + /// the beginning of the system prompt, but testing has shown it + /// to be more reliable when added to the beginning and end of the + /// system prompt. + /// + /// https://huggingface.co/google/gemma-4-12B-it#2-thinking-mode-configuration + public static func gemma4Unified( + directory: URL, + input: UserInput, + tools: [any ToolProtocol] = [], + maxInputTokenCount: Int? = nil, + maxOutputTokenCount: Int? = nil + ) throws -> LLM { + try SHLLM.assertSupportedDevice + return .init( + directory: directory, + input: input, + tools: tools, + maxInputTokenCount: maxInputTokenCount, + maxOutputTokenCount: maxOutputTokenCount, + customConfiguration: { config in + var config = config + config.extraEOSTokens = [""] + return config + }, + generateParameters: generateParameters, + responseParser: gemma4Parser + ) + } + + // https://huggingface.co/google/gemma-4-12B-it#best-practices + // + // 1. Sampling Parameters + // + // - temperature=1.0 + // - top_p=0.95 + // - top_k=64 + static var generateParameters: GenerateParameters { + GenerateParameters( + temperature: 1.0, + topP: 0.95, + topK: 64 + ) + } + + static var gemma4_12B: URL { + get throws { + let dir = "gemma-4-12B-it-4bit" + return try Bundle.shllm.directory(named: dir) + } + } +} + // MARK: - gpt-oss extension LLM where Model == GPTOSSModel { diff --git a/Sources/SHLLM/ResponseParser.swift b/Sources/SHLLM/ResponseParser.swift index 4be1f89..322e211 100644 --- a/Sources/SHLLM/ResponseParser.swift +++ b/Sources/SHLLM/ResponseParser.swift @@ -6,6 +6,8 @@ import class MLXLLM.Qwen3Model import class MLXLLM.Qwen3MoEModel import enum MLXLMCommon.Generation import struct MLXLMCommon.ToolCall +import class MLXVLM.Gemma4 +import class MLXVLM.Gemma4Unified import class MLXVLM.Mistral3VLM import class MLXVLM.Qwen35 import class MLXVLM.Qwen35MoE @@ -52,6 +54,14 @@ public extension LLM where Model == Qwen3VL { static var qwen3VLThinkingParser = defaultsToThinkingParser } +public extension LLM where Model == Gemma4 { + static var gemma4Parser: ResponseParser { Gemma4ChannelParser().parser } +} + +public extension LLM where Model == Gemma4Unified { + static var gemma4Parser: ResponseParser { Gemma4ChannelParser().parser } +} + public extension LLM where Model == Qwen35 { static func qwen3_5Parser(for input: UserInput) -> ResponseParser { qwen35Parser(for: input) @@ -242,3 +252,174 @@ private extension LLM { } } } + +// MARK: - Gemma 4 Channel Parser + +/// Parses Gemma 4's channel markers to separate reasoning from text. +/// +/// Gemma 4 models wrap their output in channels delimited by special tokens: +/// - `<|channel>thought\n` starts a thinking block (reasoning) +/// - `` ends the thinking block and starts the final response (text) +/// +/// The model always begins with `<|channel>thought\n`, so the parser starts by +/// buffering until the thought-channel header is consumed. When `` +/// is encountered, it switches to text mode. The channel markers themselves +/// are consumed and not emitted. +/// +/// Reference: https://huggingface.co/google/gemma-4-12B-it#channel-thought +final class Gemma4ChannelParser: @unchecked Sendable { + private let state = Locked(ParserState()) + + var parser: LLM.ResponseParser { + LLM.ResponseParser { (generation: Generation) -> Response? in + self.state.access { state in + state.process(generation) + } + } + } + + private struct ParserState { + private enum Mode { + case detectingHeader + case reasoning + case text + } + + private var mode = Mode.detectingHeader + private var buffer = "" + private var queuedResponses = [Response]() + + mutating func process(_ generation: Generation) -> Response? { + switch generation { + case let .chunk(chunk): + process(chunk) + + case let .toolCall(toolCall): + queuedResponses.append(.toolCall(toolCall)) + + case .info: + break + } + + return dequeue() + } + + private mutating func process(_ chunk: String) { + guard !chunk.isEmpty else { return } + + switch mode { + case .detectingHeader: + buffer += chunk + processHeaderBuffer() + + case .reasoning: + buffer += chunk + processReasoningBuffer() + + case .text: + enqueueText(chunk) + } + } + + private mutating func processHeaderBuffer() { + let thoughtHeader = Gemma4ChannelMarkers.thoughtHeader + let endTag = Gemma4ChannelMarkers.endTag + + if buffer.hasPrefix(thoughtHeader) { + let remainder = String(buffer.dropFirst(thoughtHeader.count)) + buffer = "" + mode = .reasoning + + if !remainder.isEmpty { + buffer = remainder + processReasoningBuffer() + } + return + } + + if thoughtHeader.hasPrefix(buffer) { + return + } + + if buffer.hasPrefix(endTag) { + let remainder = String(buffer.dropFirst(endTag.count)) + buffer = "" + mode = .text + enqueueText(remainder) + return + } + + if endTag.hasPrefix(buffer) { + return + } + + let text = buffer + buffer = "" + mode = .text + enqueueText(text) + } + + private mutating func processReasoningBuffer() { + let endTag = Gemma4ChannelMarkers.endTag + + if let range = buffer.range(of: endTag) { + let reasoning = String(buffer[.. 0 else { + enqueueReasoning(buffer) + buffer = "" + return + } + + let reasoningEndIndex = buffer.index(buffer.endIndex, offsetBy: -pendingLength) + let reasoning = String(buffer[.. Response? { + guard !queuedResponses.isEmpty else { return nil } + return queuedResponses.removeFirst() + } + } +} + +private enum Gemma4ChannelMarkers { + static let thoughtHeader = "<|channel>thought\n" + static let endTag = "" +} + +private extension String { + func lengthOfSuffixMatchingPrefix(of marker: String) -> Int { + let maxLength = Swift.min(count, marker.count - 1) + guard maxLength > 0 else { return 0 } + + for length in stride(from: maxLength, through: 1, by: -1) { + let suffixStartIndex = index(endIndex, offsetBy: -length) + let suffix = String(self[suffixStartIndex...]) + if marker.hasPrefix(suffix) { + return length + } + } + + return 0 + } +} diff --git a/Sources/SHLLM/SHLLM.swift b/Sources/SHLLM/SHLLM.swift index 85dd827..d1f4ef8 100644 --- a/Sources/SHLLM/SHLLM.swift +++ b/Sources/SHLLM/SHLLM.swift @@ -94,6 +94,8 @@ extension Chat.Message: @retroactive @unchecked Sendable {} @_exported import class MLXLLM.Gemma2Model @_exported import class MLXLLM.Gemma3TextModel +@_exported import class MLXLLM.Gemma4Model +@_exported import class MLXLLM.Gemma4TextModel @_exported import class MLXLLM.GemmaModel @_exported import class MLXLLM.GPTOSSModel @_exported import class MLXLLM.LFM2MoEModel @@ -110,6 +112,8 @@ extension Chat.Message: @retroactive @unchecked Sendable {} @_exported import class MLXLLM.Qwen3MoEModel @_exported import class MLXVLM.Gemma3 +@_exported import class MLXVLM.Gemma4 +@_exported import class MLXVLM.Gemma4Unified @_exported import class MLXVLM.Mistral3VLM @_exported import class MLXVLM.Qwen35 @_exported import class MLXVLM.Qwen35MoE diff --git a/Tests/SHLLMTests/Gemma4ChannelParserTests.swift b/Tests/SHLLMTests/Gemma4ChannelParserTests.swift new file mode 100644 index 0000000..27a3fb2 --- /dev/null +++ b/Tests/SHLLMTests/Gemma4ChannelParserTests.swift @@ -0,0 +1,188 @@ +import MLXLMCommon +@testable import SHLLM +import Testing + +@Suite +struct Gemma4ChannelParserTests { + @Test + func separatesReasoningFromText() { + let output = parse(chunks: [ + "<|channel>", + "thought", + "\n", + "I need to think about this.", + "", + "Here is my answer.", + ]) + + #expect(output.reasoning == "I need to think about this.") + #expect(output.text == "Here is my answer.") + #expect(output.toolCalls.isEmpty) + } + + @Test + func handlesEmptyThinkingBlock() { + let output = parse(chunks: [ + "<|channel>", + "thought", + "\n", + "", + "The price of AAPL is $123.45.", + ]) + + #expect(output.reasoning.isEmpty) + #expect(output.text == "The price of AAPL is $123.45.") + #expect(output.toolCalls.isEmpty) + } + + @Test + func passesPlainTextThroughWhenNoChannelHeader() { + let output = parse(chunks: [ + "Just a plain response without channel markers.", + ]) + + #expect(output.reasoning.isEmpty) + #expect(output.text == "Just a plain response without channel markers.") + #expect(output.toolCalls.isEmpty) + } + + @Test + func emitsBothResponsesWhenEndTagIsMergedWithFinalText() { + let output = parse(chunks: [ + "<|channel>thought\nReasoning here.Final answer.", + ]) + + #expect(output.reasoning == "Reasoning here.") + #expect(output.text == "Final answer.") + #expect(output.toolCalls.isEmpty) + } + + @Test + func handlesHeaderSplitAcrossChunks() { + let output = parse(chunks: [ + "<|channel>thought", + "\n", + "Some reasoning.", + "", + "The answer.", + ]) + + #expect(output.reasoning == "Some reasoning.") + #expect(output.text == "The answer.") + #expect(output.toolCalls.isEmpty) + } + + @Test + func handlesEndTagSplitAcrossChunks() { + let output = parse(chunks: [ + "<|channel>thought\n", + "Reasoning", + "", + "Text after split.", + ]) + + #expect(output.reasoning == "Reasoning") + #expect(output.text == "Text after split.") + #expect(output.toolCalls.isEmpty) + } + + @Test + func consumesOnlyTheHeaderNewline() { + let output = parse(chunks: [ + "<|channel>thought\n\nReasoning starts after a blank line.", + "", + "Done.", + ]) + + #expect(output.reasoning == "\nReasoning starts after a blank line.") + #expect(output.text == "Done.") + #expect(output.toolCalls.isEmpty) + } + + @Test + func handlesCharacterByCharacterMarkers() { + let chunks = Array("<|channel>thought\nThink.Answer.").map(String.init) + let output = parse(chunks: chunks) + + #expect(output.reasoning == "Think.") + #expect(output.text == "Answer.") + #expect(output.toolCalls.isEmpty) + } + + @Test + func passesToolCallsThrough() { + let toolCall = ToolCall( + function: ToolCall.Function( + name: "get_stock_price", + arguments: ["symbol": "AAPL"] + ) + ) + let output = parse(generations: [ + .chunk("<|channel>thought\n"), + .toolCall(toolCall), + .info(.test), + ]) + + #expect(output.reasoning.isEmpty) + #expect(output.text.isEmpty) + #expect(output.toolCalls == [toolCall]) + } + + @Test + func flushesQueuedFinalTextOnInfo() { + let output = parse(generations: [ + .chunk("<|channel>thought\nReasoning.Final."), + .info(.test), + ]) + + #expect(output.reasoning == "Reasoning.") + #expect(output.text == "Final.") + #expect(output.toolCalls.isEmpty) + } + + private func parse(chunks: [String]) -> ParsedOutput { + parse(generations: chunks.map(Generation.chunk) + [.info(.test)]) + } + + private func parse(generations: [Generation]) -> ParsedOutput { + let parser = LLM.gemma4Parser + var output = ParsedOutput() + + for generation in generations { + guard let response = parser.parse(generation) else { + continue + } + + switch response { + case let .reasoning(delta): + output.reasoning += delta + + case let .text(delta): + output.text += delta + + case let .toolCall(toolCall): + output.toolCalls.append(toolCall) + } + } + + return output + } + + private struct ParsedOutput { + var reasoning = "" + var text = "" + var toolCalls = [ToolCall]() + } +} + +private extension GenerateCompletionInfo { + static var test: GenerateCompletionInfo { + GenerateCompletionInfo( + promptTokenCount: 0, + generationTokenCount: 0, + promptTime: 1, + generationTime: 1 + ) + } +} diff --git a/Tests/SHLLMTests/Models/Gemma4-12BTests.swift b/Tests/SHLLMTests/Models/Gemma4-12BTests.swift new file mode 100644 index 0000000..df10132 --- /dev/null +++ b/Tests/SHLLMTests/Models/Gemma4-12BTests.swift @@ -0,0 +1,263 @@ +import Foundation +import MLXVLM +@testable import SHLLM +import Testing + +@Suite(.serialized) +struct Gemma4_12BTests { + @Test + func canStreamResult() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let input: UserInput = .init(messages: [ + ["role": "system", "content": "You are a helpful assistant."], + ["role": "user", "content": "What is the meaning of life?"], + ]) + + guard let llm = try gemma4_12B(input: input) else { return } + + var response = "" + for try await token in llm.text { + response += token + } + + Swift.print(response) + #expect(!response.isEmpty) + } + + @Test + func canAwaitResult() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let input: UserInput = .init(messages: [ + ["role": "system", "content": "You are a helpful assistant."], + ["role": "user", "content": "What is the meaning of life?"], + ]) + + guard let llm = try gemma4_12B(input: input) else { return } + + let response = try await llm.text.result + + Swift.print(response) + #expect(!response.isEmpty) + } + + @Test() + @MainActor + func canExtractTextFromImageData() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let data = try authenticationFactors + guard let llm = try gemma4_12B(image: data) else { return } + + var response = "" + for try await token in llm.text { + response += token + } + + Swift.print(response) + #expect(response.contains("The 3 authentication factors")) + #expect(response.contains("Something you forgot")) + #expect(response.contains("Something you left in the taxi")) + #expect(response.contains("Something that can be chopped off")) + } + + @Test() + @MainActor + func canExtractTextFromImageURL() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let url = try authenticationFactorsURL + guard let llm = try gemma4_12B(image: url) else { return } + + var response = "" + for try await token in llm.text { + response += token + } + + Swift.print(response) + #expect(response.contains("The 3 authentication factors")) + #expect(response.contains("Something you forgot")) + #expect(response.contains("Something you left in the taxi")) + #expect(response.contains("Something that can be chopped off")) + } + + @Test + func canFetchTheWeather() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let input = UserInput( + chat: [ + .system( + "You are a weather assistant who must use the get_current_weather tool to fetch weather data for any location the user asks about.<|think|>" + ), + .user("What is the weather in Paris, France?"), + ], + additionalContext: ["enable_thinking": true] + ) + + guard let llm = try gemma4_12B(input, tools: [weatherTool]) else { return } + + var reply = "" + var toolCallCount = 0 + var weatherLocationFound = false + + for try await response in llm { + switch response { + case .reasoning: + break + case let .text(text): + reply.append(text) + case let .toolCall(toolCall): + toolCallCount += 1 + #expect(toolCall.function.name == "get_current_weather") + + if case let .string(location) = toolCall.function.arguments["location"] { + weatherLocationFound = location.lowercased().contains("paris") + } + } + } + + #expect(reply.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + #expect(toolCallCount == 1) + #expect(weatherLocationFound) + } + + @Test + func canUseStockToolAndRespond() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let chat: [Chat.Message] = [ + .system(""" + <|think|> + You are a helpful assistant that can provide stock prices. + When asked for a stock price, you must use the get_stock_price tool. + <|think|> + """), + .user("What is the price of AAPL?"), + ] + + var input = UserInput( + chat: chat, + additionalContext: ["enable_thinking": true] + ) + + guard let llm1 = try gemma4_12B(input, tools: [stockTool]) else { return } + + var (reasoning, text, toolCallsOpt) = try await llm1.result + let toolCall = try #require(toolCallsOpt?.first) + + Swift.print("\(reasoning ?? "")\n\(text ?? "")") + #expect(reasoning != nil) + #expect(text == nil) + #expect(toolCall.function.name == "get_stock_price") + #expect(toolCall.function.arguments["symbol"] == .string("AAPL")) + + input.appendAssistantToolCall(toolCall) + input.appendToolResult(["price": 123.45]) + + guard let llm2 = try gemma4_12B(input, tools: [stockTool]) else { return } + + (reasoning, text, toolCallsOpt) = try await llm2.result + let result = try #require(text) + Swift.print("\(reasoning ?? "")\n\(result)") + #expect(!result.isEmpty) + #expect(result.lowercased().contains("aapl")) + #expect(result.contains("123.45")) + } +} + +private extension Gemma4_12BTests { + func gemma4_12B( + _ input: UserInput, + tools: [any ToolProtocol] = [] + ) throws -> LLM? { + try loadModel( + directory: LLM.gemma4_12B, + input: input, + tools: tools, + customConfiguration: { config in + var config = config + config.extraEOSTokens = [""] + return config + }, + responseParser: LLM.gemma4Parser + ) + } + + func gemma4_12B( + input: UserInput + ) throws -> LLM? { + try gemma4_12B(input, tools: []) + } + + func gemma4_12B( + image: Data + ) throws -> LLM? { + try loadModel( + directory: LLM.gemma4_12B, + input: imageInput(image), + customConfiguration: { config in + var config = config + config.extraEOSTokens = [""] + return config + }, + responseParser: LLM.gemma4Parser + ) + } + + func gemma4_12B( + image: URL + ) throws -> LLM? { + try loadModel( + directory: LLM.gemma4_12B, + input: imageInput(image), + customConfiguration: { config in + var config = config + config.extraEOSTokens = [""] + return config + }, + responseParser: LLM.gemma4Parser + ) + } + + var authenticationFactorsURL: URL { + get throws { + guard let url = Bundle.module.url( + forResource: "3-authentication-factors", + withExtension: "png" + ) else { + throw NSError( + domain: NSURLErrorDomain, + code: NSURLErrorFileDoesNotExist, + userInfo: nil + ) + } + return url + } + } + + var authenticationFactors: Data { + get throws { + try Data(contentsOf: authenticationFactorsURL) + } + } +} diff --git a/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift b/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift new file mode 100644 index 0000000..0bb8393 --- /dev/null +++ b/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift @@ -0,0 +1,263 @@ +import Foundation +import MLXVLM +@testable import SHLLM +import Testing + +@Suite(.serialized) +struct Gemma4_E2BTests { + @Test + func canStreamResult() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let input: UserInput = .init(messages: [ + ["role": "system", "content": "You are a helpful assistant."], + ["role": "user", "content": "What is the meaning of life?"], + ]) + + guard let llm = try gemma4_E2B(input: input) else { return } + + var response = "" + for try await token in llm.text { + response += token + } + + Swift.print(response) + #expect(!response.isEmpty) + } + + @Test + func canAwaitResult() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let input: UserInput = .init(messages: [ + ["role": "system", "content": "You are a helpful assistant."], + ["role": "user", "content": "What is the meaning of life?"], + ]) + + guard let llm = try gemma4_E2B(input: input) else { return } + + let response = try await llm.text.result + + Swift.print(response) + #expect(!response.isEmpty) + } + + @Test() + @MainActor + func canExtractTextFromImageData() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let data = try authenticationFactors + guard let llm = try gemma4_E2B(image: data) else { return } + + var response = "" + for try await token in llm.text { + response += token + } + + Swift.print(response) + #expect(response.contains("The 3 authentication factors")) + #expect(response.contains("Something you forgot")) + #expect(response.contains("Something you left in the taxi")) + #expect(response.contains("Something that can be chopped off")) + } + + @Test() + @MainActor + func canExtractTextFromImageURL() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let url = try authenticationFactorsURL + guard let llm = try gemma4_E2B(image: url) else { return } + + var response = "" + for try await token in llm.text { + response += token + } + + Swift.print(response) + #expect(response.contains("The 3 authentication factors")) + #expect(response.contains("Something you forgot")) + #expect(response.contains("Something you left in the taxi")) + #expect(response.contains("Something that can be chopped off")) + } + + @Test + func canFetchTheWeather() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let input = UserInput( + chat: [ + .system( + "You are a weather assistant who must use the get_current_weather tool to fetch weather data for any location the user asks about.<|think|>" + ), + .user("What is the weather in Paris, France?"), + ], + additionalContext: ["enable_thinking": true] + ) + + guard let llm = try gemma4_E2B(input, tools: [weatherTool]) else { return } + + var reply = "" + var toolCallCount = 0 + var weatherLocationFound = false + + for try await response in llm { + switch response { + case .reasoning: + break + case let .text(text): + reply.append(text) + case let .toolCall(toolCall): + toolCallCount += 1 + #expect(toolCall.function.name == "get_current_weather") + + if case let .string(location) = toolCall.function.arguments["location"] { + weatherLocationFound = location.lowercased().contains("paris") + } + } + } + + #expect(reply.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + #expect(toolCallCount == 1) + #expect(weatherLocationFound) + } + + @Test + func canUseStockToolAndRespond() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let chat: [Chat.Message] = [ + .system(""" + <|think|> + You are a helpful assistant that can provide stock prices. + When asked for a stock price, you must use the get_stock_price tool. + <|think|> + """), + .user("What is the price of AAPL?"), + ] + + var input = UserInput( + chat: chat, + additionalContext: ["enable_thinking": true] + ) + + guard let llm1 = try gemma4_E2B(input, tools: [stockTool]) else { return } + + var (reasoning, text, toolCallsOpt) = try await llm1.result + let toolCall = try #require(toolCallsOpt?.first) + + Swift.print("\(reasoning ?? "")\n\(text ?? "")") + #expect(reasoning != nil) + #expect(text == nil) + #expect(toolCall.function.name == "get_stock_price") + #expect(toolCall.function.arguments["symbol"] == .string("AAPL")) + + input.appendAssistantToolCall(toolCall) + input.appendToolResult(["price": 123.45]) + + guard let llm2 = try gemma4_E2B(input, tools: [stockTool]) else { return } + + (reasoning, text, toolCallsOpt) = try await llm2.result + let result = try #require(text) + Swift.print("\(reasoning ?? "")\n\(result)") + #expect(!result.isEmpty) + #expect(result.lowercased().contains("aapl")) + #expect(result.contains("123.45")) + } +} + +private extension Gemma4_E2BTests { + func gemma4_E2B( + _ input: UserInput, + tools: [any ToolProtocol] = [] + ) throws -> LLM? { + try loadModel( + directory: LLM.gemma4_E2B, + input: input, + tools: tools, + customConfiguration: { config in + var config = config + config.extraEOSTokens = [""] + return config + }, + responseParser: LLM.gemma4Parser + ) + } + + func gemma4_E2B( + input: UserInput + ) throws -> LLM? { + try gemma4_E2B(input, tools: []) + } + + func gemma4_E2B( + image: Data + ) throws -> LLM? { + try loadModel( + directory: LLM.gemma4_E2B, + input: imageInput(image), + customConfiguration: { config in + var config = config + config.extraEOSTokens = [""] + return config + }, + responseParser: LLM.gemma4Parser + ) + } + + func gemma4_E2B( + image: URL + ) throws -> LLM? { + try loadModel( + directory: LLM.gemma4_E2B, + input: imageInput(image), + customConfiguration: { config in + var config = config + config.extraEOSTokens = [""] + return config + }, + responseParser: LLM.gemma4Parser + ) + } + + var authenticationFactorsURL: URL { + get throws { + guard let url = Bundle.module.url( + forResource: "3-authentication-factors", + withExtension: "png" + ) else { + throw NSError( + domain: NSURLErrorDomain, + code: NSURLErrorFileDoesNotExist, + userInfo: nil + ) + } + return url + } + } + + var authenticationFactors: Data { + get throws { + try Data(contentsOf: authenticationFactorsURL) + } + } +} diff --git a/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift b/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift new file mode 100644 index 0000000..4c29441 --- /dev/null +++ b/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift @@ -0,0 +1,263 @@ +import Foundation +import MLXVLM +@testable import SHLLM +import Testing + +@Suite(.serialized) +struct Gemma4_E4BTests { + @Test + func canStreamResult() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let input: UserInput = .init(messages: [ + ["role": "system", "content": "You are a helpful assistant."], + ["role": "user", "content": "What is the meaning of life?"], + ]) + + guard let llm = try gemma4_E4B(input: input) else { return } + + var response = "" + for try await token in llm.text { + response += token + } + + Swift.print(response) + #expect(!response.isEmpty) + } + + @Test + func canAwaitResult() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let input: UserInput = .init(messages: [ + ["role": "system", "content": "You are a helpful assistant."], + ["role": "user", "content": "What is the meaning of life?"], + ]) + + guard let llm = try gemma4_E4B(input: input) else { return } + + let response = try await llm.text.result + + Swift.print(response) + #expect(!response.isEmpty) + } + + @Test() + @MainActor + func canExtractTextFromImageData() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let data = try authenticationFactors + guard let llm = try gemma4_E4B(image: data) else { return } + + var response = "" + for try await token in llm.text { + response += token + } + + Swift.print(response) + #expect(response.contains("The 3 authentication factors")) + #expect(response.contains("Something you forgot")) + #expect(response.contains("Something you left in the taxi")) + #expect(response.contains("Something that can be chopped off")) + } + + @Test() + @MainActor + func canExtractTextFromImageURL() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let url = try authenticationFactorsURL + guard let llm = try gemma4_E4B(image: url) else { return } + + var response = "" + for try await token in llm.text { + response += token + } + + Swift.print(response) + #expect(response.contains("The 3 authentication factors")) + #expect(response.contains("Something you forgot")) + #expect(response.contains("Something you left in the taxi")) + #expect(response.contains("Something that can be chopped off")) + } + + @Test + func canFetchTheWeather() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let input = UserInput( + chat: [ + .system( + "You are a weather assistant who must use the get_current_weather tool to fetch weather data for any location the user asks about.<|think|>" + ), + .user("What is the weather in Paris, France?"), + ], + additionalContext: ["enable_thinking": true] + ) + + guard let llm = try gemma4_E4B(input, tools: [weatherTool]) else { return } + + var reply = "" + var toolCallCount = 0 + var weatherLocationFound = false + + for try await response in llm { + switch response { + case .reasoning: + break + case let .text(text): + reply.append(text) + case let .toolCall(toolCall): + toolCallCount += 1 + #expect(toolCall.function.name == "get_current_weather") + + if case let .string(location) = toolCall.function.arguments["location"] { + weatherLocationFound = location.lowercased().contains("paris") + } + } + } + + #expect(reply.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + #expect(toolCallCount == 1) + #expect(weatherLocationFound) + } + + @Test + func canUseStockToolAndRespond() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let chat: [Chat.Message] = [ + .system(""" + <|think|> + You are a helpful assistant that can provide stock prices. + When asked for a stock price, you must use the get_stock_price tool. + <|think|> + """), + .user("What is the price of AAPL?"), + ] + + var input = UserInput( + chat: chat, + additionalContext: ["enable_thinking": true] + ) + + guard let llm1 = try gemma4_E4B(input, tools: [stockTool]) else { return } + + var (reasoning, text, toolCallsOpt) = try await llm1.result + let toolCall = try #require(toolCallsOpt?.first) + + Swift.print("\(reasoning ?? "")\n\(text ?? "")") + #expect(reasoning != nil) + #expect(text == nil) + #expect(toolCall.function.name == "get_stock_price") + #expect(toolCall.function.arguments["symbol"] == .string("AAPL")) + + input.appendAssistantToolCall(toolCall) + input.appendToolResult(["price": 123.45]) + + guard let llm2 = try gemma4_E4B(input, tools: [stockTool]) else { return } + + (reasoning, text, toolCallsOpt) = try await llm2.result + let result = try #require(text) + Swift.print("\(reasoning ?? "")\n\(result)") + #expect(!result.isEmpty) + #expect(result.lowercased().contains("aapl")) + #expect(result.contains("123.45")) + } +} + +private extension Gemma4_E4BTests { + func gemma4_E4B( + _ input: UserInput, + tools: [any ToolProtocol] = [] + ) throws -> LLM? { + try loadModel( + directory: LLM.gemma4_E4B, + input: input, + tools: tools, + customConfiguration: { config in + var config = config + config.extraEOSTokens = [""] + return config + }, + responseParser: LLM.gemma4Parser + ) + } + + func gemma4_E4B( + input: UserInput + ) throws -> LLM? { + try gemma4_E4B(input, tools: []) + } + + func gemma4_E4B( + image: Data + ) throws -> LLM? { + try loadModel( + directory: LLM.gemma4_E4B, + input: imageInput(image), + customConfiguration: { config in + var config = config + config.extraEOSTokens = [""] + return config + }, + responseParser: LLM.gemma4Parser + ) + } + + func gemma4_E4B( + image: URL + ) throws -> LLM? { + try loadModel( + directory: LLM.gemma4_E4B, + input: imageInput(image), + customConfiguration: { config in + var config = config + config.extraEOSTokens = [""] + return config + }, + responseParser: LLM.gemma4Parser + ) + } + + var authenticationFactorsURL: URL { + get throws { + guard let url = Bundle.module.url( + forResource: "3-authentication-factors", + withExtension: "png" + ) else { + throw NSError( + domain: NSURLErrorDomain, + code: NSURLErrorFileDoesNotExist, + userInfo: nil + ) + } + return url + } + } + + var authenticationFactors: Data { + get throws { + try Data(contentsOf: authenticationFactorsURL) + } + } +} diff --git a/bin/download.sh b/bin/download.sh index 076251c..ebdc561 100755 --- a/bin/download.sh +++ b/bin/download.sh @@ -15,6 +15,9 @@ ids=( "gemma-3-27b-it-qat-4bit" "gemma-3-4b-it-qat-3bit" "gemma-3-4b-it-qat-4bit" + "gemma-4-12B-it-4bit" + "gemma-4-e2b-it-4bit" + "gemma-4-e4b-it-4bit" "LFM2-8B-A1B-4bit" "Llama-3.2-1B-Instruct-4bit" "Llama-3.2-3B-Instruct-4bit" From 120b054d511a0665001ca116f326b88800a7d91e Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Fri, 3 Jul 2026 16:54:32 +0200 Subject: [PATCH 03/17] Pass tool schemas into generation - Bump mlx-swift-lm to 0.0.16 - Preserve input tool schemas when preparing loaded model input - Add model coverage for non-string tool arguments and final tool responses --- Package.resolved | 6 +- Package.swift | 2 +- Sources/SHLLM/LLM.swift | 4 +- .../SHLLMTests/Gemma4ChannelParserTests.swift | 1 + Tests/SHLLMTests/Helpers.swift | 27 +++++++ .../Models/Devstral2Small-24BTests.swift | 44 +++++++++++ Tests/SHLLMTests/Models/GPTOSS-20BTests.swift | 46 ++++++++++++ Tests/SHLLMTests/Models/Gemma4-12BTests.swift | 73 ++++++++++++++++--- Tests/SHLLMTests/Models/Gemma4-E2BTests.swift | 73 ++++++++++++++++--- Tests/SHLLMTests/Models/Gemma4-E4BTests.swift | 73 ++++++++++++++++--- .../SHLLMTests/Models/LFM2-8B-A1BTests.swift | 46 ++++++++++++ .../Models/Ministral-3-14BTests.swift | 44 +++++++++++ .../Models/NemotronNano-30BTests.swift | 46 ++++++++++++ .../Models/Orchestrator-8BTests.swift | 45 ++++++++++++ .../SHLLMTests/Models/Qwen2_5-1_5BTests.swift | 44 +++++++++++ Tests/SHLLMTests/Models/Qwen2_5-7BTests.swift | 44 +++++++++++ Tests/SHLLMTests/Models/Qwen3-0_6BTests.swift | 46 ++++++++++++ Tests/SHLLMTests/Models/Qwen3-1_7BTests.swift | 46 ++++++++++++ Tests/SHLLMTests/Models/Qwen3-30BTests.swift | 45 ++++++++++++ Tests/SHLLMTests/Models/Qwen3-4BTests.swift | 45 ++++++++++++ Tests/SHLLMTests/Models/Qwen3-8BTests.swift | 44 +++++++++++ .../SHLLMTests/Models/Qwen3_5-27BTests.swift | 45 ++++++++++++ Tests/SHLLMTests/Models/Qwen3_5-2BTests.swift | 48 ++++++++++++ .../Models/Qwen3_5-35B-A3BTests.swift | 45 ++++++++++++ Tests/SHLLMTests/Models/Qwen3_5-4BTests.swift | 45 ++++++++++++ Tests/SHLLMTests/Models/Qwen3_5-9BTests.swift | 45 ++++++++++++ 26 files changed, 1034 insertions(+), 38 deletions(-) diff --git a/Package.resolved b/Package.resolved index c2fb260..e535898 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "41ae51a69b4ebad5129ae3c32f2396d7aaee03fe871f45bac58696c688b8718f", + "originHash" : "249e833911a08229425856e4e4bf99018e562b937b235fb4ed690e5a9f732467", "pins" : [ { "identity" : "mlx-swift", @@ -15,8 +15,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/shareup/mlx-swift-lm", "state" : { - "revision" : "bd4b7434e6bdb588c7ef55706ff8904cb7fd4c57", - "version" : "0.0.15" + "revision" : "fa8400f8b369df8fdd845d4958339788cca83520", + "version" : "0.0.16" } }, { diff --git a/Package.swift b/Package.swift index ed22405..df2612a 100644 --- a/Package.swift +++ b/Package.swift @@ -17,7 +17,7 @@ let package = Package( ), .package( url: "https://github.com/shareup/mlx-swift-lm", - from: "0.0.15" + from: "0.0.16" ), .package( url: "https://github.com/DePasqualeOrg/swift-tokenizers", diff --git a/Sources/SHLLM/LLM.swift b/Sources/SHLLM/LLM.swift index e20a378..af2cae9 100644 --- a/Sources/SHLLM/LLM.swift +++ b/Sources/SHLLM/LLM.swift @@ -145,6 +145,7 @@ public struct LLM: AsyncSequence { } case let .loaded(context): + let toolSchemas = input.tools let input = try await context.processor.prepare(input: input) var params = generateParameters if let maxOutputTokenCount { @@ -153,7 +154,8 @@ public struct LLM: AsyncSequence { let stream = try MLXLMCommon.generate( input: input, parameters: params, - context: context + context: context, + tools: toolSchemas ) var iterator = stream.makeAsyncIterator() diff --git a/Tests/SHLLMTests/Gemma4ChannelParserTests.swift b/Tests/SHLLMTests/Gemma4ChannelParserTests.swift index 27a3fb2..7180a86 100644 --- a/Tests/SHLLMTests/Gemma4ChannelParserTests.swift +++ b/Tests/SHLLMTests/Gemma4ChannelParserTests.swift @@ -1,4 +1,5 @@ import MLXLMCommon +import MLXVLM @testable import SHLLM import Testing diff --git a/Tests/SHLLMTests/Helpers.swift b/Tests/SHLLMTests/Helpers.swift index fa1c666..67990e7 100644 --- a/Tests/SHLLMTests/Helpers.swift +++ b/Tests/SHLLMTests/Helpers.swift @@ -143,6 +143,33 @@ let stockTool = Tool( StockResponse(price: 150.0) } +struct MailReadArguments: Codable, Hashable, Sendable { + let account: String + let mailbox: String + let id: Int +} + +struct MailReadResponse: Codable, Hashable, Sendable { + let subject: String +} + +let mailReadSubject = "Your new scale is ready: Meet BodyFit" + +let mailReadTool = Tool< + MailReadArguments, + MailReadResponse +>( + name: "mail_read", + description: "Read one email by numeric message id.", + parameters: [ + .required("account", type: .string, description: "The email account name."), + .required("mailbox", type: .string, description: "The mailbox path."), + .required("id", type: .int, description: "The numeric message id."), + ] +) { _ in + MailReadResponse(subject: mailReadSubject) +} + struct NewsArguments: Codable, CustomStringConvertible, Hashable, Sendable { var query: String var sortBy: String? diff --git a/Tests/SHLLMTests/Models/Devstral2Small-24BTests.swift b/Tests/SHLLMTests/Models/Devstral2Small-24BTests.swift index c1f601d..7113811 100644 --- a/Tests/SHLLMTests/Models/Devstral2Small-24BTests.swift +++ b/Tests/SHLLMTests/Models/Devstral2Small-24BTests.swift @@ -209,6 +209,50 @@ struct Devstral2Small_24BTests { #expect(result.contains("123.45")) } + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + let chat: [Chat.Message] = [ + .system(""" + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput(chat: chat) + + guard let llm1 = try devstral2( + input, + tools: [mailReadTool] + ) else { return } + + let (_, text, toolCallsOpt) = try await llm1.result + let toolCall = try #require(toolCallsOpt?.first) + + #expect(text == nil) + #expect(toolCall.function.name == "mail_read") + #expect(toolCall.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + + guard let llm2 = try devstral2( + input, + tools: [mailReadTool] + ) else { return } + + let (_, text2, toolCallsOpt2) = try await llm2.result + let result = try #require(text2) + Swift.print(result) + #expect(result.contains(mailReadSubject)) + #expect(toolCallsOpt2 == nil) + } + @Test func canCompleteMultiToolWorkflowAndEmail() async throws { let chat: [Chat.Message] = [ diff --git a/Tests/SHLLMTests/Models/GPTOSS-20BTests.swift b/Tests/SHLLMTests/Models/GPTOSS-20BTests.swift index 1fcfcfc..3d1c25a 100644 --- a/Tests/SHLLMTests/Models/GPTOSS-20BTests.swift +++ b/Tests/SHLLMTests/Models/GPTOSS-20BTests.swift @@ -214,6 +214,52 @@ struct GPTOSS_20BTests { #expect(result.contains("123.45")) } + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + let chat: [Chat.Message] = [ + .system(""" + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput(chat: chat) + + guard let llm1 = try gptOSS_20B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning, text, toolCallsOpt) = try await llm1.result + let toolCall = try #require(toolCallsOpt?.first) + + #expect(reasoning != nil) + #expect(text == nil) + #expect(toolCall.function.name == "mail_read") + #expect(toolCall.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendHarmonyAssistantToolCall(toolCall) + input.appendHarmonyToolResult(MailReadResponse(subject: mailReadSubject)) + + guard let llm2 = try gptOSS_20B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning2, text2, toolCallsOpt2) = try await llm2.result + let result = try #require(text2) + Swift.print(result) + #expect(reasoning2 != nil) + #expect(result.contains(mailReadSubject)) + #expect(toolCallsOpt2 == nil) + } + @Test func canCompleteMultiToolWorkflowAndEmail() async throws { let chat: [Chat.Message] = [ diff --git a/Tests/SHLLMTests/Models/Gemma4-12BTests.swift b/Tests/SHLLMTests/Models/Gemma4-12BTests.swift index df10132..426ac87 100644 --- a/Tests/SHLLMTests/Models/Gemma4-12BTests.swift +++ b/Tests/SHLLMTests/Models/Gemma4-12BTests.swift @@ -162,26 +162,77 @@ struct Gemma4_12BTests { guard let llm1 = try gemma4_12B(input, tools: [stockTool]) else { return } - var (reasoning, text, toolCallsOpt) = try await llm1.result - let toolCall = try #require(toolCallsOpt?.first) + let (reasoning1, text1, toolCallsOpt1) = try await llm1.result + let toolCall1 = try #require(toolCallsOpt1?.first) - Swift.print("\(reasoning ?? "")\n\(text ?? "")") - #expect(reasoning != nil) - #expect(text == nil) - #expect(toolCall.function.name == "get_stock_price") - #expect(toolCall.function.arguments["symbol"] == .string("AAPL")) + Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") + #expect(reasoning1 != nil) + #expect(text1 == nil) + #expect(toolCall1.function.name == "get_stock_price") + #expect(toolCall1.function.arguments["symbol"] == .string("AAPL")) - input.appendAssistantToolCall(toolCall) + input.appendAssistantToolCall(toolCall1) input.appendToolResult(["price": 123.45]) guard let llm2 = try gemma4_12B(input, tools: [stockTool]) else { return } - (reasoning, text, toolCallsOpt) = try await llm2.result - let result = try #require(text) - Swift.print("\(reasoning ?? "")\n\(result)") + let (reasoning2, text2, toolCallsOpt2) = try await llm2.result + let result = try #require(text2) + Swift.print("\(reasoning2 ?? "")\n\(result)") #expect(!result.isEmpty) #expect(result.lowercased().contains("aapl")) #expect(result.contains("123.45")) + #expect(toolCallsOpt2 == nil) + } + + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let chat: [Chat.Message] = [ + .system(""" + <|think|> + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + <|think|> + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput( + chat: chat, + additionalContext: ["enable_thinking": true] + ) + + guard let llm1 = try gemma4_12B(input, tools: [mailReadTool]) else { return } + + let (reasoning1, text1, toolCallsOpt1) = try await llm1.result + let toolCall1 = try #require(toolCallsOpt1?.first) + + Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") + #expect(reasoning1 != nil) + #expect(text1 == nil) + #expect(toolCall1.function.name == "mail_read") + #expect(toolCall1.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall1.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall1.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall1) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + + guard let llm2 = try gemma4_12B(input, tools: [mailReadTool]) else { return } + + let (reasoning2, text2, toolCallsOpt2) = try await llm2.result + let result = try #require(text2) + Swift.print("\(reasoning2 ?? "")\n\(result)") + #expect(result.contains(mailReadSubject)) + #expect(toolCallsOpt2 == nil) } } diff --git a/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift b/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift index 0bb8393..ea62dff 100644 --- a/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift +++ b/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift @@ -162,26 +162,77 @@ struct Gemma4_E2BTests { guard let llm1 = try gemma4_E2B(input, tools: [stockTool]) else { return } - var (reasoning, text, toolCallsOpt) = try await llm1.result - let toolCall = try #require(toolCallsOpt?.first) + let (reasoning1, text1, toolCallsOpt1) = try await llm1.result + let toolCall1 = try #require(toolCallsOpt1?.first) - Swift.print("\(reasoning ?? "")\n\(text ?? "")") - #expect(reasoning != nil) - #expect(text == nil) - #expect(toolCall.function.name == "get_stock_price") - #expect(toolCall.function.arguments["symbol"] == .string("AAPL")) + Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") + #expect(reasoning1 != nil) + #expect(text1 == nil) + #expect(toolCall1.function.name == "get_stock_price") + #expect(toolCall1.function.arguments["symbol"] == .string("AAPL")) - input.appendAssistantToolCall(toolCall) + input.appendAssistantToolCall(toolCall1) input.appendToolResult(["price": 123.45]) guard let llm2 = try gemma4_E2B(input, tools: [stockTool]) else { return } - (reasoning, text, toolCallsOpt) = try await llm2.result - let result = try #require(text) - Swift.print("\(reasoning ?? "")\n\(result)") + let (reasoning2, text2, toolCallsOpt2) = try await llm2.result + let result = try #require(text2) + Swift.print("\(reasoning2 ?? "")\n\(result)") #expect(!result.isEmpty) #expect(result.lowercased().contains("aapl")) #expect(result.contains("123.45")) + #expect(toolCallsOpt2 == nil) + } + + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let chat: [Chat.Message] = [ + .system(""" + <|think|> + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + <|think|> + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput( + chat: chat, + additionalContext: ["enable_thinking": true] + ) + + guard let llm1 = try gemma4_E2B(input, tools: [mailReadTool]) else { return } + + let (reasoning1, text1, toolCallsOpt1) = try await llm1.result + let toolCall1 = try #require(toolCallsOpt1?.first) + + Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") + #expect(reasoning1 != nil) + #expect(text1 == nil) + #expect(toolCall1.function.name == "mail_read") + #expect(toolCall1.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall1.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall1.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall1) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + + guard let llm2 = try gemma4_E2B(input, tools: [mailReadTool]) else { return } + + let (reasoning2, text2, toolCallsOpt2) = try await llm2.result + let result = try #require(text2) + Swift.print("\(reasoning2 ?? "")\n\(result)") + #expect(result.contains(mailReadSubject)) + #expect(toolCallsOpt2 == nil) } } diff --git a/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift b/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift index 4c29441..b978eaf 100644 --- a/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift +++ b/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift @@ -162,26 +162,77 @@ struct Gemma4_E4BTests { guard let llm1 = try gemma4_E4B(input, tools: [stockTool]) else { return } - var (reasoning, text, toolCallsOpt) = try await llm1.result - let toolCall = try #require(toolCallsOpt?.first) + let (reasoning1, text1, toolCallsOpt1) = try await llm1.result + let toolCall1 = try #require(toolCallsOpt1?.first) - Swift.print("\(reasoning ?? "")\n\(text ?? "")") - #expect(reasoning != nil) - #expect(text == nil) - #expect(toolCall.function.name == "get_stock_price") - #expect(toolCall.function.arguments["symbol"] == .string("AAPL")) + Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") + #expect(reasoning1 != nil) + #expect(text1 == nil) + #expect(toolCall1.function.name == "get_stock_price") + #expect(toolCall1.function.arguments["symbol"] == .string("AAPL")) - input.appendAssistantToolCall(toolCall) + input.appendAssistantToolCall(toolCall1) input.appendToolResult(["price": 123.45]) guard let llm2 = try gemma4_E4B(input, tools: [stockTool]) else { return } - (reasoning, text, toolCallsOpt) = try await llm2.result - let result = try #require(text) - Swift.print("\(reasoning ?? "")\n\(result)") + let (reasoning2, text2, toolCallsOpt2) = try await llm2.result + let result = try #require(text2) + Swift.print("\(reasoning2 ?? "")\n\(result)") #expect(!result.isEmpty) #expect(result.lowercased().contains("aapl")) #expect(result.contains("123.45")) + #expect(toolCallsOpt2 == nil) + } + + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + guard SHLLM.isSupportedDevice else { + Swift.print("⚠️ Metal GPU not available") + return + } + + let chat: [Chat.Message] = [ + .system(""" + <|think|> + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + <|think|> + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput( + chat: chat, + additionalContext: ["enable_thinking": true] + ) + + guard let llm1 = try gemma4_E4B(input, tools: [mailReadTool]) else { return } + + let (reasoning1, text1, toolCallsOpt1) = try await llm1.result + let toolCall1 = try #require(toolCallsOpt1?.first) + + Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") + #expect(reasoning1 != nil) + #expect(text1 == nil) + #expect(toolCall1.function.name == "mail_read") + #expect(toolCall1.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall1.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall1.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall1) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + + guard let llm2 = try gemma4_E4B(input, tools: [mailReadTool]) else { return } + + let (reasoning2, text2, toolCallsOpt2) = try await llm2.result + let result = try #require(text2) + Swift.print("\(reasoning2 ?? "")\n\(result)") + #expect(result.contains(mailReadSubject)) + #expect(toolCallsOpt2 == nil) } } diff --git a/Tests/SHLLMTests/Models/LFM2-8B-A1BTests.swift b/Tests/SHLLMTests/Models/LFM2-8B-A1BTests.swift index f1a9805..15c8190 100644 --- a/Tests/SHLLMTests/Models/LFM2-8B-A1BTests.swift +++ b/Tests/SHLLMTests/Models/LFM2-8B-A1BTests.swift @@ -163,6 +163,52 @@ struct LFM2_8B_A1BTests { #expect(result.contains("123.45")) } + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + let chat: [Chat.Message] = [ + .system(""" + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput(chat: chat) + + guard let llm1 = try lfm2_8B_A1B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning, text, toolCallsOpt) = try await llm1.result + let toolCall = try #require(toolCallsOpt?.first) + + #expect(reasoning == nil) + #expect(text == nil) + #expect(toolCall.function.name == "mail_read") + #expect(toolCall.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + + guard let llm2 = try lfm2_8B_A1B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning2, text2, toolCallsOpt2) = try await llm2.result + let result = try #require(text2) + Swift.print(result) + #expect(reasoning2 == nil) + #expect(result.contains(mailReadSubject)) + #expect(toolCallsOpt2 == nil) + } + @Test() func canCompleteMultiToolWorkflowAndEmail() async throws { let chat: [Chat.Message] = [ diff --git a/Tests/SHLLMTests/Models/Ministral-3-14BTests.swift b/Tests/SHLLMTests/Models/Ministral-3-14BTests.swift index f754fee..b63c4f0 100644 --- a/Tests/SHLLMTests/Models/Ministral-3-14BTests.swift +++ b/Tests/SHLLMTests/Models/Ministral-3-14BTests.swift @@ -209,6 +209,50 @@ struct Ministral_3_14BTests { #expect(result.contains("123.45")) } + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + let chat: [Chat.Message] = [ + .system(""" + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput(chat: chat) + + guard let llm1 = try ministral( + input, + tools: [mailReadTool] + ) else { return } + + let (_, text, toolCallsOpt) = try await llm1.result + let toolCall = try #require(toolCallsOpt?.first) + + #expect(text == nil) + #expect(toolCall.function.name == "mail_read") + #expect(toolCall.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + + guard let llm2 = try ministral( + input, + tools: [mailReadTool] + ) else { return } + + let (_, text2, toolCallsOpt2) = try await llm2.result + let result = try #require(text2) + Swift.print(result) + #expect(result.contains(mailReadSubject)) + #expect(toolCallsOpt2 == nil) + } + @Test func canCompleteMultiToolWorkflowAndEmail() async throws { let chat: [Chat.Message] = [ diff --git a/Tests/SHLLMTests/Models/NemotronNano-30BTests.swift b/Tests/SHLLMTests/Models/NemotronNano-30BTests.swift index de67b9b..1af7ddc 100644 --- a/Tests/SHLLMTests/Models/NemotronNano-30BTests.swift +++ b/Tests/SHLLMTests/Models/NemotronNano-30BTests.swift @@ -215,6 +215,52 @@ struct NemotronNano_30BTests { #expect(result.contains("123.45")) } + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + let chat: [Chat.Message] = [ + .system(""" + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput(chat: chat) + + guard let llm1 = try nemotronNano( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning, text, toolCallsOpt) = try await llm1.result + let toolCall = try #require(toolCallsOpt?.first) + + #expect(reasoning != nil) + #expect(text == nil) + #expect(toolCall.function.name == "mail_read") + #expect(toolCall.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + + guard let llm2 = try nemotronNano( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning2, text2, toolCallsOpt2) = try await llm2.result + let result = try #require(text2) + Swift.print(result) + #expect(reasoning2 != nil) + #expect(result.contains(mailReadSubject)) + #expect(toolCallsOpt2 == nil) + } + @Test func canCompleteMultiToolWorkflowAndEmail() async throws { let chat: [Chat.Message] = [ diff --git a/Tests/SHLLMTests/Models/Orchestrator-8BTests.swift b/Tests/SHLLMTests/Models/Orchestrator-8BTests.swift index 990e4d9..c8e8c5b 100644 --- a/Tests/SHLLMTests/Models/Orchestrator-8BTests.swift +++ b/Tests/SHLLMTests/Models/Orchestrator-8BTests.swift @@ -169,6 +169,51 @@ struct Orchestrator_8BTests { #expect(result.contains("123.45")) } + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + let chat: [Chat.Message] = [ + .system(""" + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput(chat: chat) + + guard let llm1 = try orchestrator_8B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning, text, toolCallsOpt) = try await llm1.result + let toolCall = try #require(toolCallsOpt?.first) + + #expect(reasoning != nil) + #expect(text == nil) + #expect(toolCall.function.name == "mail_read") + #expect(toolCall.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + + guard let llm2 = try orchestrator_8B( + input, + tools: [mailReadTool] + ) else { return } + + let (_, text2, toolCallsOpt2) = try await llm2.result + let result = try #require(text2) + Swift.print(result) + #expect(result.contains(mailReadSubject)) + #expect(toolCallsOpt2 == nil) + } + @Test(.disabled()) func canCompleteMultiToolWorkflowAndEmail() async throws { let chat: [Chat.Message] = [ diff --git a/Tests/SHLLMTests/Models/Qwen2_5-1_5BTests.swift b/Tests/SHLLMTests/Models/Qwen2_5-1_5BTests.swift index 3464ea3..e38efcb 100644 --- a/Tests/SHLLMTests/Models/Qwen2_5-1_5BTests.swift +++ b/Tests/SHLLMTests/Models/Qwen2_5-1_5BTests.swift @@ -76,6 +76,50 @@ struct Qwen2_5__1_5BTests { #expect(toolCallCount == 1) #expect(weatherLocationFound) } + + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + let chat: [Chat.Message] = [ + .system(""" + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput(chat: chat) + + guard let llm1 = try qwen2_5__1_5B( + input, + tools: [mailReadTool] + ) else { return } + + let (_, text, toolCallsOpt) = try await llm1.result + let toolCall = try #require(toolCallsOpt?.first) + + Swift.print(text ?? "") + #expect(toolCall.function.name == "mail_read") + #expect(toolCall.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + + guard let llm2 = try qwen2_5__1_5B( + input, + tools: [mailReadTool] + ) else { return } + + let (_, text2, toolCallsOpt2) = try await llm2.result + let result = try #require(text2) + Swift.print(result) + #expect(result.contains(mailReadSubject)) + #expect(toolCallsOpt2 == nil) + } } private func qwen2_5__1_5B( diff --git a/Tests/SHLLMTests/Models/Qwen2_5-7BTests.swift b/Tests/SHLLMTests/Models/Qwen2_5-7BTests.swift index 8f4c1c1..9b1d520 100644 --- a/Tests/SHLLMTests/Models/Qwen2_5-7BTests.swift +++ b/Tests/SHLLMTests/Models/Qwen2_5-7BTests.swift @@ -76,6 +76,50 @@ struct Qwen2_5__7BTests { #expect(toolCallCount == 1) #expect(weatherLocationFound) } + + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + let chat: [Chat.Message] = [ + .system(""" + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput(chat: chat) + + guard let llm1 = try qwen2_5__7B( + input, + tools: [mailReadTool] + ) else { return } + + let (_, text, toolCallsOpt) = try await llm1.result + let toolCall = try #require(toolCallsOpt?.first) + + Swift.print(text ?? "") + #expect(toolCall.function.name == "mail_read") + #expect(toolCall.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + + guard let llm2 = try qwen2_5__7B( + input, + tools: [mailReadTool] + ) else { return } + + let (_, text2, toolCallsOpt2) = try await llm2.result + let result = try #require(text2) + Swift.print(result) + #expect(result.contains(mailReadSubject)) + #expect(toolCallsOpt2 == nil) + } } private func qwen2_5__7B( diff --git a/Tests/SHLLMTests/Models/Qwen3-0_6BTests.swift b/Tests/SHLLMTests/Models/Qwen3-0_6BTests.swift index f2e20e2..cb8f712 100644 --- a/Tests/SHLLMTests/Models/Qwen3-0_6BTests.swift +++ b/Tests/SHLLMTests/Models/Qwen3-0_6BTests.swift @@ -130,6 +130,52 @@ struct Qwen3__0_6BTests { #expect(toolCallCount >= 1) #expect(weatherLocationFound) } + + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + let chat: [Chat.Message] = [ + .system(""" + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput(chat: chat) + + guard let llm1 = try qwen3__0_6B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning, text, toolCallsOpt) = try await llm1.result + let toolCall = try #require(toolCallsOpt?.first) + + #expect(reasoning != nil) + #expect(text == nil) + #expect(toolCall.function.name == "mail_read") + #expect(toolCall.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + + guard let llm2 = try qwen3__0_6B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning2, text2, toolCallsOpt2) = try await llm2.result + let result = try #require(text2) + Swift.print(result) + #expect(reasoning2 != nil) + #expect(result.contains(mailReadSubject)) + #expect(toolCallsOpt2 == nil) + } } private func qwen3__0_6B( diff --git a/Tests/SHLLMTests/Models/Qwen3-1_7BTests.swift b/Tests/SHLLMTests/Models/Qwen3-1_7BTests.swift index 2a1c5dc..61d4b44 100644 --- a/Tests/SHLLMTests/Models/Qwen3-1_7BTests.swift +++ b/Tests/SHLLMTests/Models/Qwen3-1_7BTests.swift @@ -129,6 +129,52 @@ struct Qwen3__1_7BTests { #expect(toolCallCount == 1) #expect(weatherLocationFound) } + + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + let chat: [Chat.Message] = [ + .system(""" + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput(chat: chat) + + guard let llm1 = try qwen3__1_7B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning, text, toolCallsOpt) = try await llm1.result + let toolCall = try #require(toolCallsOpt?.first) + + #expect(reasoning != nil) + #expect(text == nil) + #expect(toolCall.function.name == "mail_read") + #expect(toolCall.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + + guard let llm2 = try qwen3__1_7B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning2, text2, toolCallsOpt2) = try await llm2.result + let result = try #require(text2) + Swift.print(result) + #expect(reasoning2 != nil) + #expect(result.contains(mailReadSubject)) + #expect(toolCallsOpt2 == nil) + } } private func qwen3__1_7B( diff --git a/Tests/SHLLMTests/Models/Qwen3-30BTests.swift b/Tests/SHLLMTests/Models/Qwen3-30BTests.swift index 44d2f7b..9f940bc 100644 --- a/Tests/SHLLMTests/Models/Qwen3-30BTests.swift +++ b/Tests/SHLLMTests/Models/Qwen3-30BTests.swift @@ -169,6 +169,51 @@ struct Qwen3_30BTests { #expect(result.contains("123.45")) } + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + let chat: [Chat.Message] = [ + .system(""" + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput(chat: chat) + + guard let llm1 = try qwen3MoE( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning, text, toolCallsOpt) = try await llm1.result + let toolCall = try #require(toolCallsOpt?.first) + + #expect(reasoning != nil) + #expect(text == nil) + #expect(toolCall.function.name == "mail_read") + #expect(toolCall.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + + guard let llm2 = try qwen3MoE( + input, + tools: [mailReadTool] + ) else { return } + + let (_, text2, toolCallsOpt2) = try await llm2.result + let result = try #require(text2) + Swift.print(result) + #expect(result.contains(mailReadSubject)) + #expect(toolCallsOpt2 == nil) + } + @Test func canCompleteMultiToolWorkflowAndEmail() async throws { let chat: [Chat.Message] = [ diff --git a/Tests/SHLLMTests/Models/Qwen3-4BTests.swift b/Tests/SHLLMTests/Models/Qwen3-4BTests.swift index 8fb3458..dc86de8 100644 --- a/Tests/SHLLMTests/Models/Qwen3-4BTests.swift +++ b/Tests/SHLLMTests/Models/Qwen3-4BTests.swift @@ -169,6 +169,51 @@ struct Qwen3_4BTests { #expect(result.contains("123.45")) } + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + let chat: [Chat.Message] = [ + .system(""" + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput(chat: chat) + + guard let llm1 = try qwen3_4B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning, text, toolCallsOpt) = try await llm1.result + let toolCall = try #require(toolCallsOpt?.first) + + #expect(reasoning != nil) + #expect(text == nil) + #expect(toolCall.function.name == "mail_read") + #expect(toolCall.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + + guard let llm2 = try qwen3_4B( + input, + tools: [mailReadTool] + ) else { return } + + let (_, text2, toolCallsOpt2) = try await llm2.result + let result = try #require(text2) + Swift.print(result) + #expect(result.contains(mailReadSubject)) + #expect(toolCallsOpt2 == nil) + } + @Test func canCompleteMultiToolWorkflowAndEmail() async throws { let chat: [Chat.Message] = [ diff --git a/Tests/SHLLMTests/Models/Qwen3-8BTests.swift b/Tests/SHLLMTests/Models/Qwen3-8BTests.swift index 005eae6..72db831 100644 --- a/Tests/SHLLMTests/Models/Qwen3-8BTests.swift +++ b/Tests/SHLLMTests/Models/Qwen3-8BTests.swift @@ -219,6 +219,50 @@ struct Qwen3_8BTests { #expect(toolCallsOpt2 == nil) } + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + let chat: [Chat.Message] = [ + .system(""" + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput(chat: chat) + + guard let llm1 = try qwen3_8B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning1, text1, toolCallsOpt1) = try await llm1.result + #expect(reasoning1 != nil) + #expect(text1 == nil) + let toolCall1 = try #require(toolCallsOpt1?.first) + + #expect(toolCall1.function.name == "mail_read") + #expect(toolCall1.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall1.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall1.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall1) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + guard let llm2 = try qwen3_8B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning2, text2, toolCallsOpt2) = try await llm2.result + Swift.print(text2 ?? "") + #expect(reasoning2 != nil) + #expect(text2?.contains(mailReadSubject) == true) + #expect(toolCallsOpt2 == nil) + } + @Test func canCompleteMultiToolWorkflowAndEmail() async throws { let chat: [Chat.Message] = [ diff --git a/Tests/SHLLMTests/Models/Qwen3_5-27BTests.swift b/Tests/SHLLMTests/Models/Qwen3_5-27BTests.swift index 983c6d9..1cf963b 100644 --- a/Tests/SHLLMTests/Models/Qwen3_5-27BTests.swift +++ b/Tests/SHLLMTests/Models/Qwen3_5-27BTests.swift @@ -234,6 +234,51 @@ struct Qwen3_5_27BTests { #expect(toolCallsOpt2 == nil) } + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + let chat: [Chat.Message] = [ + .system(""" + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput(chat: chat) + + guard let llm1 = try qwen3_5__27B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning1, text1, toolCallsOpt1) = try await llm1.result + Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") + #expect(reasoning1 != nil) + #expect(text1 == nil) + let toolCall1 = try #require(toolCallsOpt1?.first) + + #expect(toolCall1.function.name == "mail_read") + #expect(toolCall1.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall1.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall1.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall1) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + guard let llm2 = try qwen3_5__27B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning2, text2, toolCallsOpt2) = try await llm2.result + Swift.print("\(reasoning2 ?? "")\n\(text2 ?? "")") + #expect(reasoning2 != nil) + #expect(text2?.contains(mailReadSubject) == true) + #expect(toolCallsOpt2 == nil) + } + @Test func canCompleteMultiToolWorkflowAndEmail() async throws { let chat: [Chat.Message] = [ diff --git a/Tests/SHLLMTests/Models/Qwen3_5-2BTests.swift b/Tests/SHLLMTests/Models/Qwen3_5-2BTests.swift index 4cbfaed..2c3831b 100644 --- a/Tests/SHLLMTests/Models/Qwen3_5-2BTests.swift +++ b/Tests/SHLLMTests/Models/Qwen3_5-2BTests.swift @@ -239,6 +239,54 @@ struct Qwen3_5_2BTests { #expect(toolCallsOpt2 == nil) } + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + let chat: [Chat.Message] = [ + .system(""" + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput( + chat: chat, + additionalContext: ["enable_thinking": false] + ) + + guard let llm1 = try qwen3_5__2B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning1, text1, toolCallsOpt1) = try await llm1.result + Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") + #expect(reasoning1 == nil) + #expect(text1 == nil) + let toolCall1 = try #require(toolCallsOpt1?.first) + + #expect(toolCall1.function.name == "mail_read") + #expect(toolCall1.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall1.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall1.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall1) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + guard let llm2 = try qwen3_5__2B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning2, text2, toolCallsOpt2) = try await llm2.result + Swift.print("\(reasoning2 ?? "")\n\(text2 ?? "")") + #expect(reasoning2 == nil) + #expect(text2?.contains(mailReadSubject) == true) + #expect(toolCallsOpt2 == nil) + } + @Test func canCompleteMultiToolWorkflowAndEmail() async throws { let chat: [Chat.Message] = [ diff --git a/Tests/SHLLMTests/Models/Qwen3_5-35B-A3BTests.swift b/Tests/SHLLMTests/Models/Qwen3_5-35B-A3BTests.swift index 4f41b17..363fe75 100644 --- a/Tests/SHLLMTests/Models/Qwen3_5-35B-A3BTests.swift +++ b/Tests/SHLLMTests/Models/Qwen3_5-35B-A3BTests.swift @@ -234,6 +234,51 @@ struct Qwen3_5_35B_A3BTests { #expect(toolCallsOpt2 == nil) } + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + let chat: [Chat.Message] = [ + .system(""" + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput(chat: chat) + + guard let llm1 = try qwen3_5MoE( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning1, text1, toolCallsOpt1) = try await llm1.result + Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") + #expect(reasoning1 != nil) + #expect(text1 == nil) + let toolCall1 = try #require(toolCallsOpt1?.first) + + #expect(toolCall1.function.name == "mail_read") + #expect(toolCall1.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall1.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall1.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall1) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + guard let llm2 = try qwen3_5MoE( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning2, text2, toolCallsOpt2) = try await llm2.result + Swift.print("\(reasoning2 ?? "")\n\(text2 ?? "")") + #expect(reasoning2 != nil) + #expect(text2?.contains(mailReadSubject) == true) + #expect(toolCallsOpt2 == nil) + } + @Test func canCompleteMultiToolWorkflowAndEmail() async throws { let chat: [Chat.Message] = [ diff --git a/Tests/SHLLMTests/Models/Qwen3_5-4BTests.swift b/Tests/SHLLMTests/Models/Qwen3_5-4BTests.swift index fed0a8e..1c97bfb 100644 --- a/Tests/SHLLMTests/Models/Qwen3_5-4BTests.swift +++ b/Tests/SHLLMTests/Models/Qwen3_5-4BTests.swift @@ -234,6 +234,51 @@ struct Qwen3_5_4BTests { #expect(toolCallsOpt2 == nil) } + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + let chat: [Chat.Message] = [ + .system(""" + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput(chat: chat) + + guard let llm1 = try qwen3_5__4B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning1, text1, toolCallsOpt1) = try await llm1.result + Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") + #expect(reasoning1 != nil) + #expect(text1 == nil) + let toolCall1 = try #require(toolCallsOpt1?.first) + + #expect(toolCall1.function.name == "mail_read") + #expect(toolCall1.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall1.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall1.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall1) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + guard let llm2 = try qwen3_5__4B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning2, text2, toolCallsOpt2) = try await llm2.result + Swift.print("\(reasoning2 ?? "")\n\(text2 ?? "")") + #expect(reasoning2 != nil) + #expect(text2?.contains(mailReadSubject) == true) + #expect(toolCallsOpt2 == nil) + } + @Test func canCompleteMultiToolWorkflowAndEmail() async throws { let chat: [Chat.Message] = [ diff --git a/Tests/SHLLMTests/Models/Qwen3_5-9BTests.swift b/Tests/SHLLMTests/Models/Qwen3_5-9BTests.swift index 4cfb4b4..9a05d9b 100644 --- a/Tests/SHLLMTests/Models/Qwen3_5-9BTests.swift +++ b/Tests/SHLLMTests/Models/Qwen3_5-9BTests.swift @@ -234,6 +234,51 @@ struct Qwen3_5_9BTests { #expect(toolCallsOpt2 == nil) } + @Test + func canUseToolsWithNonStringArgumentsAndRespond() async throws { + let chat: [Chat.Message] = [ + .system(""" + You are an email assistant. When asked to read an email, call mail_read exactly once. + After the tool result is provided, reply with the email subject exactly and do not call tools again. + """), + .user("Read email 158348 from account me@example.com in mailbox INBOX."), + ] + + var input = UserInput(chat: chat) + + guard let llm1 = try qwen3_5__9B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning1, text1, toolCallsOpt1) = try await llm1.result + Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") + #expect(reasoning1 != nil) + #expect(text1 == nil) + let toolCall1 = try #require(toolCallsOpt1?.first) + + #expect(toolCall1.function.name == "mail_read") + #expect(toolCall1.function.arguments["account"] == .string("me@example.com")) + #expect(toolCall1.function.arguments["mailbox"] == .string("INBOX")) + + let idArgument = try #require(toolCall1.function.arguments["id"]) + #expect(idArgument == .int(158_348)) + #expect(idArgument != .string("158348")) + + input.appendAssistantToolCall(toolCall1) + input.appendToolResult(MailReadResponse(subject: mailReadSubject)) + guard let llm2 = try qwen3_5__9B( + input, + tools: [mailReadTool] + ) else { return } + + let (reasoning2, text2, toolCallsOpt2) = try await llm2.result + Swift.print("\(reasoning2 ?? "")\n\(text2 ?? "")") + #expect(reasoning2 != nil) + #expect(text2?.contains(mailReadSubject) == true) + #expect(toolCallsOpt2 == nil) + } + @Test func canCompleteMultiToolWorkflowAndEmail() async throws { let chat: [Chat.Message] = [ From 5d29ba52cd5a8eb65057f8caec9c192ed726d7ce Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Fri, 3 Jul 2026 18:20:56 +0200 Subject: [PATCH 04/17] Run xcodebuild directly instead of inside eval --- bin/test.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/test.sh b/bin/test.sh index 16b870c..5905907 100755 --- a/bin/test.sh +++ b/bin/test.sh @@ -33,14 +33,14 @@ if [ "$xcodeVersion" -ge 26 ]; then echo "❌ Metal toolchain is not installed" echo "⬇️ Downloading Metal toolchain..." - eval "exec xcodebuild \ + xcodebuild \ -downloadComponent metalToolchain \ - -exportPath /tmp/metalToolchainDownload/" + -exportPath /tmp/metalToolchainDownload/ echo "🧰 Installing Metal toolchain..." - eval "exec xcodebuild \ + xcodebuild \ -importComponent metalToolchain \ - -importPath /tmp/metalToolchainDownload/*.exportedBundle" + -importPath /tmp/metalToolchainDownload/*.exportedBundle fi fi @@ -60,4 +60,4 @@ else ${signingFlags} \ "${testSpecifiers[@]}" \ test -fi \ No newline at end of file +fi From 5e7cc554423eec976ec826800804be75d80c5ced Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Fri, 3 Jul 2026 18:21:50 +0200 Subject: [PATCH 05/17] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- Tests/SHLLMTests/Models/Gemma4-E4BTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift b/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift index b978eaf..948a4de 100644 --- a/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift +++ b/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift @@ -165,7 +165,7 @@ struct Gemma4_E4BTests { let (reasoning1, text1, toolCallsOpt1) = try await llm1.result let toolCall1 = try #require(toolCallsOpt1?.first) - Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") + Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") #expect(reasoning1 != nil) #expect(text1 == nil) #expect(toolCall1.function.name == "get_stock_price") From fe9dec5e3b93467f8b29fa23cbb54099024b4045 Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Fri, 3 Jul 2026 18:22:11 +0200 Subject: [PATCH 06/17] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- Tests/SHLLMTests/Models/Gemma4-E4BTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift b/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift index 948a4de..bb76300 100644 --- a/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift +++ b/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift @@ -178,7 +178,7 @@ struct Gemma4_E4BTests { let (reasoning2, text2, toolCallsOpt2) = try await llm2.result let result = try #require(text2) - Swift.print("\(reasoning2 ?? "")\n\(result)") + Swift.print("\(reasoning2 ?? "")\n\(result)") #expect(!result.isEmpty) #expect(result.lowercased().contains("aapl")) #expect(result.contains("123.45")) From 83c0227921d2fc1eb5702df36a4662986f060e9d Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Fri, 3 Jul 2026 18:22:20 +0200 Subject: [PATCH 07/17] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- Tests/SHLLMTests/Models/Gemma4-E4BTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift b/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift index bb76300..7317002 100644 --- a/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift +++ b/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift @@ -212,7 +212,7 @@ struct Gemma4_E4BTests { let (reasoning1, text1, toolCallsOpt1) = try await llm1.result let toolCall1 = try #require(toolCallsOpt1?.first) - Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") + Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") #expect(reasoning1 != nil) #expect(text1 == nil) #expect(toolCall1.function.name == "mail_read") From 1d9ed4553f113a1f0f81ce3e4373d715bf41a087 Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Fri, 3 Jul 2026 18:22:30 +0200 Subject: [PATCH 08/17] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- Tests/SHLLMTests/Models/Gemma4-E4BTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift b/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift index 7317002..003ecaa 100644 --- a/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift +++ b/Tests/SHLLMTests/Models/Gemma4-E4BTests.swift @@ -230,7 +230,7 @@ struct Gemma4_E4BTests { let (reasoning2, text2, toolCallsOpt2) = try await llm2.result let result = try #require(text2) - Swift.print("\(reasoning2 ?? "")\n\(result)") + Swift.print("\(reasoning2 ?? "")\n\(result)") #expect(result.contains(mailReadSubject)) #expect(toolCallsOpt2 == nil) } From 93cf0efcd9e66078fafea7683bd5d523e480c2fd Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Fri, 3 Jul 2026 18:22:42 +0200 Subject: [PATCH 09/17] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- Tests/SHLLMTests/Models/Gemma4-E2BTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift b/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift index ea62dff..1aee875 100644 --- a/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift +++ b/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift @@ -165,7 +165,7 @@ struct Gemma4_E2BTests { let (reasoning1, text1, toolCallsOpt1) = try await llm1.result let toolCall1 = try #require(toolCallsOpt1?.first) - Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") + Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") #expect(reasoning1 != nil) #expect(text1 == nil) #expect(toolCall1.function.name == "get_stock_price") From 764171b23d389a7b29292a3a6308e58466e98998 Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Fri, 3 Jul 2026 18:22:51 +0200 Subject: [PATCH 10/17] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- Tests/SHLLMTests/Models/Gemma4-E2BTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift b/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift index 1aee875..67537a1 100644 --- a/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift +++ b/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift @@ -230,7 +230,7 @@ struct Gemma4_E2BTests { let (reasoning2, text2, toolCallsOpt2) = try await llm2.result let result = try #require(text2) - Swift.print("\(reasoning2 ?? "")\n\(result)") + Swift.print("\(reasoning2 ?? "")\n\(result)") #expect(result.contains(mailReadSubject)) #expect(toolCallsOpt2 == nil) } From c9700695db4d4418c690dec919230e526f98ebcd Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Fri, 3 Jul 2026 18:23:00 +0200 Subject: [PATCH 11/17] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- Tests/SHLLMTests/Models/Gemma4-12BTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/SHLLMTests/Models/Gemma4-12BTests.swift b/Tests/SHLLMTests/Models/Gemma4-12BTests.swift index 426ac87..143c668 100644 --- a/Tests/SHLLMTests/Models/Gemma4-12BTests.swift +++ b/Tests/SHLLMTests/Models/Gemma4-12BTests.swift @@ -165,7 +165,7 @@ struct Gemma4_12BTests { let (reasoning1, text1, toolCallsOpt1) = try await llm1.result let toolCall1 = try #require(toolCallsOpt1?.first) - Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") + Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") #expect(reasoning1 != nil) #expect(text1 == nil) #expect(toolCall1.function.name == "get_stock_price") From 092ff98539826bafcb3bb8dad686f10709bef944 Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Fri, 3 Jul 2026 18:23:10 +0200 Subject: [PATCH 12/17] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- Tests/SHLLMTests/Models/Gemma4-12BTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/SHLLMTests/Models/Gemma4-12BTests.swift b/Tests/SHLLMTests/Models/Gemma4-12BTests.swift index 143c668..9ae96ac 100644 --- a/Tests/SHLLMTests/Models/Gemma4-12BTests.swift +++ b/Tests/SHLLMTests/Models/Gemma4-12BTests.swift @@ -178,7 +178,7 @@ struct Gemma4_12BTests { let (reasoning2, text2, toolCallsOpt2) = try await llm2.result let result = try #require(text2) - Swift.print("\(reasoning2 ?? "")\n\(result)") + Swift.print("\(reasoning2 ?? "")\n\(result)") #expect(!result.isEmpty) #expect(result.lowercased().contains("aapl")) #expect(result.contains("123.45")) From a4d0de18f559a1d09b23d04ac1323445f86086b8 Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Fri, 3 Jul 2026 18:23:17 +0200 Subject: [PATCH 13/17] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- Tests/SHLLMTests/Models/Gemma4-12BTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/SHLLMTests/Models/Gemma4-12BTests.swift b/Tests/SHLLMTests/Models/Gemma4-12BTests.swift index 9ae96ac..0b2afdb 100644 --- a/Tests/SHLLMTests/Models/Gemma4-12BTests.swift +++ b/Tests/SHLLMTests/Models/Gemma4-12BTests.swift @@ -212,7 +212,7 @@ struct Gemma4_12BTests { let (reasoning1, text1, toolCallsOpt1) = try await llm1.result let toolCall1 = try #require(toolCallsOpt1?.first) - Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") + Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") #expect(reasoning1 != nil) #expect(text1 == nil) #expect(toolCall1.function.name == "mail_read") From f9dd717f5e1ffca9b01e9bc739323f61e5968c71 Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Fri, 3 Jul 2026 18:23:24 +0200 Subject: [PATCH 14/17] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- Tests/SHLLMTests/Models/Gemma4-12BTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/SHLLMTests/Models/Gemma4-12BTests.swift b/Tests/SHLLMTests/Models/Gemma4-12BTests.swift index 0b2afdb..c90011f 100644 --- a/Tests/SHLLMTests/Models/Gemma4-12BTests.swift +++ b/Tests/SHLLMTests/Models/Gemma4-12BTests.swift @@ -230,7 +230,7 @@ struct Gemma4_12BTests { let (reasoning2, text2, toolCallsOpt2) = try await llm2.result let result = try #require(text2) - Swift.print("\(reasoning2 ?? "")\n\(result)") + Swift.print("\(reasoning2 ?? "")\n\(result)") #expect(result.contains(mailReadSubject)) #expect(toolCallsOpt2 == nil) } From c3af6a1d3fa833adee4f40f008c7ff653be2e821 Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Fri, 3 Jul 2026 18:23:36 +0200 Subject: [PATCH 15/17] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- Tests/SHLLMTests/Models/Gemma4-E2BTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift b/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift index 67537a1..41bf045 100644 --- a/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift +++ b/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift @@ -178,7 +178,7 @@ struct Gemma4_E2BTests { let (reasoning2, text2, toolCallsOpt2) = try await llm2.result let result = try #require(text2) - Swift.print("\(reasoning2 ?? "")\n\(result)") + Swift.print("\(reasoning2 ?? "")\n\(result)") #expect(!result.isEmpty) #expect(result.lowercased().contains("aapl")) #expect(result.contains("123.45")) From 2f95eca3603596c21ca7dbafd1dd06f614f7e674 Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Fri, 3 Jul 2026 18:23:47 +0200 Subject: [PATCH 16/17] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- Tests/SHLLMTests/Models/Gemma4-E2BTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift b/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift index 41bf045..f77337c 100644 --- a/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift +++ b/Tests/SHLLMTests/Models/Gemma4-E2BTests.swift @@ -212,7 +212,7 @@ struct Gemma4_E2BTests { let (reasoning1, text1, toolCallsOpt1) = try await llm1.result let toolCall1 = try #require(toolCallsOpt1?.first) - Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") + Swift.print("\(reasoning1 ?? "")\n\(text1 ?? "")") #expect(reasoning1 != nil) #expect(text1 == nil) #expect(toolCall1.function.name == "mail_read") From d9faa594f4e5a8cdb9efd24c392faa862fbe23fa Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Fri, 3 Jul 2026 18:28:32 +0200 Subject: [PATCH 17/17] Flush Gemma parser buffer on info events --- Sources/SHLLM/ResponseParser.swift | 18 +++++++++++++- .../SHLLMTests/Gemma4ChannelParserTests.swift | 24 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/Sources/SHLLM/ResponseParser.swift b/Sources/SHLLM/ResponseParser.swift index 322e211..f6e4f27 100644 --- a/Sources/SHLLM/ResponseParser.swift +++ b/Sources/SHLLM/ResponseParser.swift @@ -298,7 +298,7 @@ final class Gemma4ChannelParser: @unchecked Sendable { queuedResponses.append(.toolCall(toolCall)) case .info: - break + flushBuffer() } return dequeue() @@ -385,6 +385,22 @@ final class Gemma4ChannelParser: @unchecked Sendable { enqueueReasoning(reasoning) } + private mutating func flushBuffer() { + guard !buffer.isEmpty else { return } + + let text = buffer + buffer = "" + + switch mode { + case .detectingHeader, .text: + mode = .text + enqueueText(text) + + case .reasoning: + enqueueReasoning(text) + } + } + private mutating func enqueueReasoning(_ reasoning: String) { guard !reasoning.isEmpty else { return } queuedResponses.append(.reasoning(reasoning)) diff --git a/Tests/SHLLMTests/Gemma4ChannelParserTests.swift b/Tests/SHLLMTests/Gemma4ChannelParserTests.swift index 7180a86..028c29b 100644 --- a/Tests/SHLLMTests/Gemma4ChannelParserTests.swift +++ b/Tests/SHLLMTests/Gemma4ChannelParserTests.swift @@ -142,6 +142,30 @@ struct Gemma4ChannelParserTests { #expect(output.toolCalls.isEmpty) } + @Test + func flushesPartialEndTagOnInfo() { + let output = parse(generations: [ + .chunk("<|channel>thought\nReasoningthought"), + .info(.test), + ]) + + #expect(output.reasoning.isEmpty) + #expect(output.text == "<|channel>thought") + #expect(output.toolCalls.isEmpty) + } + private func parse(chunks: [String]) -> ParsedOutput { parse(generations: chunks.map(Generation.chunk) + [.info(.test)]) }