From b4904765cf432d73c6e71c39972656ecace6a3a6 Mon Sep 17 00:00:00 2001 From: "tattn (Tatsuya Tanaka)" Date: Tue, 25 Aug 2026 22:29:04 +0900 Subject: [PATCH] Fix CI to enhance stability by optimizing thread usage and handling pending tokens in LlamaClient --- .github/workflows/test.yml | 9 +-- Package.swift | 41 +++++++------- Sources/LocalLLMClientLlama/Context.swift | 16 +++++- Sources/LocalLLMClientLlama/LlamaClient.swift | 2 + Sources/LocalLLMClientLlamaC/include/utils.h | 2 + .../LLMSessionLlamaTests.swift | 55 ++++++++++--------- .../LocalLLMClientLlamaTests.swift | 16 ++++++ .../LocalLLMClientLlamaTests/ModelTests.swift | 2 +- 8 files changed, 93 insertions(+), 50 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ad9e37e..7ed9938 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,6 +37,7 @@ jobs: env: DEVELOPER_DIR: "/Applications/Xcode_26.4.app/Contents/Developer" TEST_RUNNER_GITHUB_MODEL_CACHE: "${{ github.workspace }}/model_cache" + TEST_RUNNER_GGML_METAL_DEVICES: "0" steps: - &checkout uses: actions/checkout@v4 @@ -100,7 +101,7 @@ jobs: working-directory: Example run: | xcodebuild -downloadPlatform iOS - xcodebuild build -skipMacroValidation -project LocalLLMClientExample.xcodeproj -scheme LocalLLMClientExample -destination 'platform=iOS Simulator,name=iPhone 17 Pro,OS=26.4' CODE_SIGN_IDENTITY="-" + xcodebuild build -skipMacroValidation -project LocalLLMClientExample.xcodeproj -scheme LocalLLMClientExample -destination 'generic/platform=iOS Simulator' CODE_SIGN_IDENTITY="-" test-ubuntu-x86_64: runs-on: ubuntu-latest @@ -137,10 +138,10 @@ jobs: mkdir -p ${{ github.workspace }}/lib # Download and extract llama.cpp binaries - LLAMA_URL="https://github.com/ggml-org/llama.cpp/releases/download/${LLAMA_VERSION}/llama-${LLAMA_VERSION}-bin-ubuntu-x64.zip" + LLAMA_URL="https://github.com/ggml-org/llama.cpp/releases/download/${LLAMA_VERSION}/llama-${LLAMA_VERSION}-bin-ubuntu-x64.tar.gz" echo "Downloading llama.cpp binaries from: $LLAMA_URL" - curl -L $LLAMA_URL -o llama-bin.zip - unzip -j llama-bin.zip "*.so" -d "${{ github.workspace }}/lib" + curl -fL $LLAMA_URL -o llama-bin.tar.gz + tar -xzf llama-bin.tar.gz -C "${{ github.workspace }}/lib" --strip-components=1 --wildcards '*/*.so*' ls -la ${{ github.workspace }}/lib - name: Build package diff --git a/Package.swift b/Package.swift index a2e9655..68331a3 100644 --- a/Package.swift +++ b/Package.swift @@ -45,6 +45,24 @@ packageProducts.append(contentsOf: [ ]) #endif +// MARK: - llama.cpp Target Settings + +// Shared by the Apple and Linux definitions of LocalLLMClientLlamaC so they cannot drift apart. +let llamaCSettings: [CSetting] = [ + .unsafeFlags(["-w"]), + .define("LLAMA_BUILD_NUMBER", to: llamaBuildNumber), + .headerSearchPath("."), + .headerSearchPath("common") +] + +// mtmd-audio.cpp declares `constexpr bool DEBUG`, which a `DEBUG` macro would break. +let llamaCxxSettings: [CXXSetting] = [ + .unsafeFlags(["-UDEBUG"]), + .define("LLAMA_BUILD_NUMBER", to: llamaBuildNumber), + .headerSearchPath("."), + .headerSearchPath("common") +] + // MARK: - Package Targets var packageTargets: [Target] = [ @@ -168,18 +186,8 @@ packageTargets.append(contentsOf: [ name: "LocalLLMClientLlamaC", dependencies: ["LocalLLMClientLlamaFramework"], exclude: ["exclude"], - cSettings: [ - .unsafeFlags(["-w"]), - .define("LLAMA_BUILD_NUMBER", to: llamaBuildNumber), - .headerSearchPath("."), - .headerSearchPath("common") - ], - cxxSettings: [ - .unsafeFlags(["-UDEBUG"]), - .define("LLAMA_BUILD_NUMBER", to: llamaBuildNumber), - .headerSearchPath("."), - .headerSearchPath("common") - ], + cSettings: llamaCSettings, + cxxSettings: llamaCxxSettings, swiftSettings: [ .interoperabilityMode(.Cxx) ] @@ -239,13 +247,8 @@ packageTargets.append(contentsOf: [ .target( name: "LocalLLMClientLlamaC", exclude: ["exclude"], - cSettings: [ - .unsafeFlags(["-w"]), - .headerSearchPath(".") - ], - cxxSettings: [ - .headerSearchPath(".") - ], + cSettings: llamaCSettings, + cxxSettings: llamaCxxSettings, swiftSettings: [ .interoperabilityMode(.Cxx) ], diff --git a/Sources/LocalLLMClientLlama/Context.swift b/Sources/LocalLLMClientLlama/Context.swift index b0811b6..756dfd6 100644 --- a/Sources/LocalLLMClientLlama/Context.swift +++ b/Sources/LocalLLMClientLlama/Context.swift @@ -36,12 +36,18 @@ public final class Context: @unchecked Sendable { return llama_memory_seq_pos_max(kv, 0) + 1 } + /// Keeps a couple of cores free on larger machines without starving small ones. + private static var defaultNumberOfThreads: Int { + let cores = ProcessInfo.processInfo.activeProcessorCount + return cores > 4 ? min(8, cores - 2) : max(1, cores) + } + public init(url: URL, parameter: LlamaClient.Parameter = .default) throws(LLMError) { initializeLlama() var ctx_params = llama_context_default_params() ctx_params.n_ctx = UInt32(parameter.context) - ctx_params.n_threads = Int32(parameter.numberOfThreads ?? max(1, min(8, ProcessInfo.processInfo.processorCount - 2))) + ctx_params.n_threads = Int32(parameter.numberOfThreads ?? Self.defaultNumberOfThreads) ctx_params.n_threads_batch = ctx_params.n_threads self.parameter = parameter @@ -93,6 +99,8 @@ public final class Context: @unchecked Sendable { } public func clear() { + discardPendingTokens() + guard let kv = llama_get_memory(context) else { return } @@ -100,6 +108,12 @@ public final class Context: @unchecked Sendable { llama_memory_clear(kv, true) } + /// Drops tokens a generation left pending when it ended before decoding them. + /// Appending the next prompt on top of them would write past the batch allocation. + func discardPendingTokens() { + batch.clear() + } + func addCache(for chunk: MessageChunk, position: llama_pos) { let endIndex = promptCaches.endIndex - 1 switch (chunk, promptCaches.last?.chunk) { diff --git a/Sources/LocalLLMClientLlama/LlamaClient.swift b/Sources/LocalLLMClientLlama/LlamaClient.swift index 14e0320..fc37f2a 100644 --- a/Sources/LocalLLMClientLlama/LlamaClient.swift +++ b/Sources/LocalLLMClientLlama/LlamaClient.swift @@ -57,6 +57,8 @@ public final class LlamaClient: LLMClient { /// - Returns: A generator that produces text as it's generated by the model. /// - Throws: An `LLMError.failedToDecode` error if the input cannot be decoded. public func textStream(from input: LLMInput) throws -> Generator { + context.discardPendingTokens() + do { switch input.value { case .plain(let text): diff --git a/Sources/LocalLLMClientLlamaC/include/utils.h b/Sources/LocalLLMClientLlamaC/include/utils.h index 90124d8..1376748 100644 --- a/Sources/LocalLLMClientLlamaC/include/utils.h +++ b/Sources/LocalLLMClientLlamaC/include/utils.h @@ -1,3 +1,5 @@ +#pragma once + #include #include "../common/chat.h" diff --git a/Tests/LocalLLMClientLlamaTests/LLMSessionLlamaTests.swift b/Tests/LocalLLMClientLlamaTests/LLMSessionLlamaTests.swift index a667835..320b9f0 100644 --- a/Tests/LocalLLMClientLlamaTests/LLMSessionLlamaTests.swift +++ b/Tests/LocalLLMClientLlamaTests/LLMSessionLlamaTests.swift @@ -6,7 +6,7 @@ import LocalLLMClientTestUtilities import LocalLLMClientUtility extension ModelTests { - @Suite(.serialized, .timeLimit(.minutes(5))) + @Suite(.serialized, .timeLimit(.minutes(10))) struct LLMSessionLlamaTests { private static func makeGeneralModel(size: LocalLLMClient.ModelSize = .default) -> LLMSession.DownloadModel { @@ -16,7 +16,13 @@ extension ModelTests { private static func makeToolModel(size: LocalLLMClient.ModelSize = .default) -> LLMSession.DownloadModel { let info = LocalLLMClient.modelInfo(for: .tool, modelSize: size) - return .llama(id: info.id, model: info.model, mmproj: info.clip, parameter: .init(context: 2500)) + // Pinned sampling, so tool calling does not depend on the luck of the draw. + return .llama( + id: info.id, + model: info.model, + mmproj: info.clip, + parameter: .init(context: 2500, seed: 0, temperature: 0.1) + ) } @Test @@ -78,40 +84,39 @@ extension ModelTests { } @Test - func toolCallWithMultipleTools() async throws { - // Create test tools + func calculatorToolCallWithMultipleTools() async throws { let weatherTool = TestWeatherTool() let calculatorTool = TestCalculatorTool() - + let session = LLMSession( model: Self.makeToolModel(), tools: [weatherTool, calculatorTool] ) - - // Test calculator - weatherTool.reset() - calculatorTool.reset() - - let calcResponse = try await session.respond(to: "What is 2 + 2? use calculate") - print("Calculator response: \(calcResponse)") - + + let response = try await session.respond(to: "What is 2 + 2? use calculate") + print("Calculator response: \(response)") + #expect(calculatorTool.invocationCount > 0, "Calculator tool should have been called") #expect(weatherTool.invocationCount == 0, "Weather tool should not have been called for calculation") - #expect(calcResponse.contains("4"), "Response should contain the result") - - // Test weather - weatherTool.reset() - calculatorTool.reset() + #expect(response.contains("4"), "Response should contain the result") + } + + @Test + func weatherToolCallWithMultipleTools() async throws { + let weatherTool = TestWeatherTool() + let calculatorTool = TestCalculatorTool() + + let session = LLMSession( + model: Self.makeToolModel(), + tools: [weatherTool, calculatorTool] + ) + + let response = try await session.respond(to: "What's the weather in Paris? use get_weather") + print("Weather response: \(response)") - let weatherResponse = try await session.respond(to: "What's the weather in Paris? use get_weather") - print("Weather response: \(weatherResponse)") - #expect(weatherTool.invocationCount > 0, "Weather tool should have been called") #expect(calculatorTool.invocationCount == 0, "Calculator tool should not have been called for weather") - - if let lastArgs = weatherTool.lastArguments { - #expect(lastArgs.location.lowercased().contains("paris"), "Tool should have been called with Paris as location") - } + #expect(weatherTool.lastArguments?.location.lowercased().contains("paris") == true, "Tool should have been called with Paris as location") } @Test diff --git a/Tests/LocalLLMClientLlamaTests/LocalLLMClientLlamaTests.swift b/Tests/LocalLLMClientLlamaTests/LocalLLMClientLlamaTests.swift index 6534a9a..3f4d598 100644 --- a/Tests/LocalLLMClientLlamaTests/LocalLLMClientLlamaTests.swift +++ b/Tests/LocalLLMClientLlamaTests/LocalLLMClientLlamaTests.swift @@ -95,6 +95,22 @@ extension ModelTests.LocalLLMClientLlamaTests { Issue.record() } + @Test + func decodeAfterCancelledGeneration() async throws { + let client = try await LocalLLMClient.llama(parameter: .init(context: 512, batch: 8)) + + let task = Task { + for try await _ in try await client.textStream(from: prompt) {} + } + try await Task.sleep(for: .seconds(3)) + task.cancel() + try? await task.value + + // The token the cancelled run left pending must not end up in the next prompt. + let result = try await client.generateText(from: String(repeating: "Hello, world! ", count: 40)) + #expect(!result.isEmpty) + } + @Test func overflowBatchSize() async throws { let result = try await LocalLLMClient.llama(parameter: .init(context: 512, batch: 2, options: .init(verbose: true))).generateText(from: "Hello, world!") diff --git a/Tests/LocalLLMClientLlamaTests/ModelTests.swift b/Tests/LocalLLMClientLlamaTests/ModelTests.swift index 60bb1f3..bcfddbb 100644 --- a/Tests/LocalLLMClientLlamaTests/ModelTests.swift +++ b/Tests/LocalLLMClientLlamaTests/ModelTests.swift @@ -113,7 +113,7 @@ extension LocalLLMClient { } } -@Suite(.serialized, .timeLimit(.minutes(5)), .disabled(if: disabledTests)) +@Suite(.serialized, .timeLimit(.minutes(10)), .disabled(if: disabledTests)) actor ModelTests { nonisolated(unsafe) private static var initialized = false