Fix CI

.github/workflows/test.yml

@@ -37,6 +37,7 @@ jobs:
     env:
       DEVELOPER_DIR: "/Applications/Xcode_26.4.app/Contents/Developer"
       TEST_RUNNER_GITHUB_MODEL_CACHE: "${{ github.workspace }}/model_cache"
+      TEST_RUNNER_GGML_METAL_DEVICES: "0"
     steps:
       - &checkout
         uses: actions/checkout@v4
@@ -100,7 +101,7 @@ jobs:
         working-directory: Example
         run: |
           xcodebuild -downloadPlatform iOS
-          xcodebuild build -skipMacroValidation -project LocalLLMClientExample.xcodeproj -scheme LocalLLMClientExample -destination 'platform=iOS Simulator,name=iPhone 17 Pro,OS=26.4' CODE_SIGN_IDENTITY="-"
+          xcodebuild build -skipMacroValidation -project LocalLLMClientExample.xcodeproj -scheme LocalLLMClientExample -destination 'generic/platform=iOS Simulator' CODE_SIGN_IDENTITY="-"
 
   test-ubuntu-x86_64:
     runs-on: ubuntu-latest
@@ -137,10 +138,10 @@ jobs:
           mkdir -p ${{ github.workspace }}/lib
 
           # Download and extract llama.cpp binaries
-          LLAMA_URL="https://github.com/ggml-org/llama.cpp/releases/download/${LLAMA_VERSION}/llama-${LLAMA_VERSION}-bin-ubuntu-x64.zip"
+          LLAMA_URL="https://github.com/ggml-org/llama.cpp/releases/download/${LLAMA_VERSION}/llama-${LLAMA_VERSION}-bin-ubuntu-x64.tar.gz"
           echo "Downloading llama.cpp binaries from: $LLAMA_URL"
-          curl -L $LLAMA_URL -o llama-bin.zip
-          unzip -j llama-bin.zip "*.so" -d "${{ github.workspace }}/lib"
+          curl -fL $LLAMA_URL -o llama-bin.tar.gz
+          tar -xzf llama-bin.tar.gz -C "${{ github.workspace }}/lib" --strip-components=1 --wildcards '*/*.so*'
           ls -la ${{ github.workspace }}/lib
 
       - name: Build package

Package.swift

@@ -45,6 +45,24 @@ packageProducts.append(contentsOf: [
 ])
 #endif
 
+// MARK: - llama.cpp Target Settings
+
+// Shared by the Apple and Linux definitions of LocalLLMClientLlamaC so they cannot drift apart.
+let llamaCSettings: [CSetting] = [
+    .unsafeFlags(["-w"]),
+    .define("LLAMA_BUILD_NUMBER", to: llamaBuildNumber),
+    .headerSearchPath("."),
+    .headerSearchPath("common")
+]
+
+// mtmd-audio.cpp declares `constexpr bool DEBUG`, which a `DEBUG` macro would break.
+let llamaCxxSettings: [CXXSetting] = [
+    .unsafeFlags(["-UDEBUG"]),
+    .define("LLAMA_BUILD_NUMBER", to: llamaBuildNumber),
+    .headerSearchPath("."),
+    .headerSearchPath("common")
+]
+
 // MARK: - Package Targets
 
 var packageTargets: [Target] = [
@@ -168,18 +186,8 @@ packageTargets.append(contentsOf: [
         name: "LocalLLMClientLlamaC",
         dependencies: ["LocalLLMClientLlamaFramework"],
         exclude: ["exclude"],
-        cSettings: [
-            .unsafeFlags(["-w"]),
-            .define("LLAMA_BUILD_NUMBER", to: llamaBuildNumber),
-            .headerSearchPath("."),
-            .headerSearchPath("common")
-        ],
-        cxxSettings: [
-            .unsafeFlags(["-UDEBUG"]),
-            .define("LLAMA_BUILD_NUMBER", to: llamaBuildNumber),
-            .headerSearchPath("."),
-            .headerSearchPath("common")
-        ],
+        cSettings: llamaCSettings,
+        cxxSettings: llamaCxxSettings,
         swiftSettings: [
             .interoperabilityMode(.Cxx)
         ]
@@ -239,13 +247,8 @@ packageTargets.append(contentsOf: [
     .target(
         name: "LocalLLMClientLlamaC",
         exclude: ["exclude"],
-        cSettings: [
-            .unsafeFlags(["-w"]),
-            .headerSearchPath(".")
-        ],
-        cxxSettings: [
-            .headerSearchPath(".")
-        ],
+        cSettings: llamaCSettings,
+        cxxSettings: llamaCxxSettings,
         swiftSettings: [
             .interoperabilityMode(.Cxx)
         ],

Sources/LocalLLMClientLlama/Context.swift

@@ -36,12 +36,18 @@ public final class Context: @unchecked Sendable {
         return llama_memory_seq_pos_max(kv, 0) + 1
     }
 
+    /// Keeps a couple of cores free on larger machines without starving small ones.
+    private static var defaultNumberOfThreads: Int {
+        let cores = ProcessInfo.processInfo.activeProcessorCount
+        return cores > 4 ? min(8, cores - 2) : max(1, cores)
+    }
+
     public init(url: URL, parameter: LlamaClient.Parameter = .default) throws(LLMError) {
         initializeLlama()
 
         var ctx_params = llama_context_default_params()
         ctx_params.n_ctx = UInt32(parameter.context)
-        ctx_params.n_threads = Int32(parameter.numberOfThreads ?? max(1, min(8, ProcessInfo.processInfo.processorCount - 2)))
+        ctx_params.n_threads = Int32(parameter.numberOfThreads ?? Self.defaultNumberOfThreads)
         ctx_params.n_threads_batch = ctx_params.n_threads
 
         self.parameter = parameter
@@ -93,13 +99,21 @@ public final class Context: @unchecked Sendable {
     }
 
     public func clear() {
+        discardPendingTokens()
+
         guard let kv = llama_get_memory(context) else {
             return
         }
 
         llama_memory_clear(kv, true)
     }
 
+    /// Drops tokens a generation left pending when it ended before decoding them.
+    /// Appending the next prompt on top of them would write past the batch allocation.
+    func discardPendingTokens() {
+        batch.clear()
+    }
+
     func addCache(for chunk: MessageChunk, position: llama_pos) {
         let endIndex = promptCaches.endIndex - 1
         switch (chunk, promptCaches.last?.chunk) {

Sources/LocalLLMClientLlama/LlamaClient.swift

@@ -57,6 +57,8 @@ public final class LlamaClient: LLMClient {
     /// - Returns: A generator that produces text as it's generated by the model.
     /// - Throws: An `LLMError.failedToDecode` error if the input cannot be decoded.
     public func textStream(from input: LLMInput) throws -> Generator {
+        context.discardPendingTokens()
+
         do {
             switch input.value {
             case .plain(let text):

Sources/LocalLLMClientLlamaC/include/utils.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #include <memory>
 
 #include "../common/chat.h"

Tests/LocalLLMClientLlamaTests/LLMSessionLlamaTests.swift

@@ -6,7 +6,7 @@ import LocalLLMClientTestUtilities
 import LocalLLMClientUtility
 
 extension ModelTests {
-    @Suite(.serialized, .timeLimit(.minutes(5)))
+    @Suite(.serialized, .timeLimit(.minutes(10)))
     struct LLMSessionLlamaTests {
 
         private static func makeGeneralModel(size: LocalLLMClient.ModelSize = .default) -> LLMSession.DownloadModel {
@@ -16,7 +16,13 @@ extension ModelTests {
 
         private static func makeToolModel(size: LocalLLMClient.ModelSize = .default) -> LLMSession.DownloadModel {
             let info = LocalLLMClient.modelInfo(for: .tool, modelSize: size)
-            return .llama(id: info.id, model: info.model, mmproj: info.clip, parameter: .init(context: 2500))
+            // Pinned sampling, so tool calling does not depend on the luck of the draw.
+            return .llama(
+                id: info.id,
+                model: info.model,
+                mmproj: info.clip,
+                parameter: .init(context: 2500, seed: 0, temperature: 0.1)
+            )
         }
 
         @Test
@@ -78,40 +84,39 @@ extension ModelTests {
         }
 
         @Test
-        func toolCallWithMultipleTools() async throws {
-            // Create test tools
+        func calculatorToolCallWithMultipleTools() async throws {
             let weatherTool = TestWeatherTool()
             let calculatorTool = TestCalculatorTool()
-            
+
             let session = LLMSession(
                 model: Self.makeToolModel(),
                 tools: [weatherTool, calculatorTool]
             )
-
-            // Test calculator
-            weatherTool.reset()
-            calculatorTool.reset()
-
-            let calcResponse = try await session.respond(to: "What is 2 + 2? use calculate")
-            print("Calculator response: \(calcResponse)")
-
+
+            let response = try await session.respond(to: "What is 2 + 2? use calculate")
+            print("Calculator response: \(response)")
+
             #expect(calculatorTool.invocationCount > 0, "Calculator tool should have been called")
             #expect(weatherTool.invocationCount == 0, "Weather tool should not have been called for calculation")
-            #expect(calcResponse.contains("4"), "Response should contain the result")
-
-            // Test weather
-            weatherTool.reset()
-            calculatorTool.reset()
+            #expect(response.contains("4"), "Response should contain the result")
+        }
+
+        @Test
+        func weatherToolCallWithMultipleTools() async throws {
+            let weatherTool = TestWeatherTool()
+            let calculatorTool = TestCalculatorTool()
+
+            let session = LLMSession(
+                model: Self.makeToolModel(),
+                tools: [weatherTool, calculatorTool]
+            )
+
+            let response = try await session.respond(to: "What's the weather in Paris? use get_weather")
+            print("Weather response: \(response)")
 
-            let weatherResponse = try await session.respond(to: "What's the weather in Paris? use get_weather")
-            print("Weather response: \(weatherResponse)")
-
             #expect(weatherTool.invocationCount > 0, "Weather tool should have been called")
             #expect(calculatorTool.invocationCount == 0, "Calculator tool should not have been called for weather")
-
-            if let lastArgs = weatherTool.lastArguments {
-                #expect(lastArgs.location.lowercased().contains("paris"), "Tool should have been called with Paris as location")
-            }
+            #expect(weatherTool.lastArguments?.location.lowercased().contains("paris") == true, "Tool should have been called with Paris as location")
         }
 
         @Test

Tests/LocalLLMClientLlamaTests/LocalLLMClientLlamaTests.swift

@@ -95,6 +95,22 @@ extension ModelTests.LocalLLMClientLlamaTests {
         Issue.record()
     }
 
+    @Test
+    func decodeAfterCancelledGeneration() async throws {
+        let client = try await LocalLLMClient.llama(parameter: .init(context: 512, batch: 8))
+
+        let task = Task {
+            for try await _ in try await client.textStream(from: prompt) {}
+        }
+        try await Task.sleep(for: .seconds(3))
+        task.cancel()
+        try? await task.value
+
+        // The token the cancelled run left pending must not end up in the next prompt.
+        let result = try await client.generateText(from: String(repeating: "Hello, world! ", count: 40))
+        #expect(!result.isEmpty)
+    }
+
     @Test
     func overflowBatchSize() async throws {
         let result = try await LocalLLMClient.llama(parameter: .init(context: 512, batch: 2, options: .init(verbose: true))).generateText(from: "Hello, world!")

Tests/LocalLLMClientLlamaTests/ModelTests.swift

@@ -113,7 +113,7 @@ extension LocalLLMClient {
     }
 }
 
-@Suite(.serialized, .timeLimit(.minutes(5)), .disabled(if: disabledTests))
+@Suite(.serialized, .timeLimit(.minutes(10)), .disabled(if: disabledTests))
 actor ModelTests {
     nonisolated(unsafe) private static var initialized = false