From 3e168e39c4ae968175fcea546b057cc0df5a7799 Mon Sep 17 00:00:00 2001 From: james-333i Date: Tue, 25 Aug 2026 11:29:46 -0700 Subject: [PATCH 1/2] Fix LlamaLanguageModel build against current llama.swift The open-ended dependency range resolves llama.swift to releases wrapping current llama.cpp builds, where the Llama trait no longer compiles: llama_sampler_init_penalties regained its leading n_vocab parameter, and llama_model_params replaced use_mmap and use_mlock with a llama_load_mode enum. Pass the vocabulary size at all three penalties call sites and set load_mode to LLAMA_LOAD_MODE_MMAP, matching the previous mmap-only behavior. Verified against llama.swift 2.10549.0 with the full live test suite. --- Package.resolved | 9 +++++++++ Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift | 6 ++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Package.resolved b/Package.resolved index b689a691..0b86bc46 100644 --- a/Package.resolved +++ b/Package.resolved @@ -19,6 +19,15 @@ "version" : "1.3.1" } }, + { + "identity" : "llama.swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/mattt/llama.swift", + "state" : { + "revision" : "716419d4d7aa542fce301e809cde7234c68ddbc6", + "version" : "2.10549.0" + } + }, { "identity" : "partialjsondecoder", "kind" : "remoteSourceControl", diff --git a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift index 6bcd8045..56d858a9 100644 --- a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift @@ -673,8 +673,7 @@ import Foundation params.n_gpu_layers = 0 // Try to reduce memory usage - params.use_mmap = true - params.use_mlock = false + params.load_mode = LLAMA_LOAD_MODE_MMAP return params } @@ -829,6 +828,7 @@ import Foundation llama_sampler_chain_add( samplerPtr, llama_sampler_init_penalties( + llama_vocab_n_tokens(vocab), effectiveRepeatLastN, effectiveRepeatPenalty, effectiveFrequencyPenalty, @@ -958,6 +958,7 @@ import Foundation llama_sampler_chain_add( samplerPointer, llama_sampler_init_penalties( + llama_vocab_n_tokens(vocab), options.repeatLastN, options.repeatPenalty, options.frequencyPenalty, @@ -1197,6 +1198,7 @@ import Foundation llama_sampler_chain_add( samplerPtr, llama_sampler_init_penalties( + llama_vocab_n_tokens(vocab), effectiveRepeatLastN, effectiveRepeatPenalty, effectiveFrequencyPenalty, From 2843193fdf9c1c680f548bee931d2db49fb5f1be Mon Sep 17 00:00:00 2001 From: james-333i Date: Tue, 25 Aug 2026 11:31:25 -0700 Subject: [PATCH 2/2] Enable Metal GPU offload for llama.cpp models by default LlamaLanguageModel forced n_gpu_layers = 0 with a note about Metal GPU issues. That predates the current packaging: llama.swift now wraps the official llama.cpp XCFramework, which is built with GGML_METAL=ON and GGML_METAL_EMBED_LIBRARY=ON, so the Metal backend and its shader library ship inside the binary. With full offload, the live test suite passes and runs the suite about 4x faster on Apple silicon (16.1s to 4.1s for 25 tests with a 135M Q8_0 model), with about 20x less CPU time. Add a gpuLayers parameter to the initializer, defaulting to all layers on real hardware and to CPU-only in the simulator, where Metal execution remains unreliable for llama.cpp. Pass 0 to keep the previous CPU-only behavior. --- .../Models/LlamaLanguageModel.swift | 28 ++++++++++++++++--- .../LlamaLanguageModelTests.swift | 4 +++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift index 56d858a9..fc73ea50 100644 --- a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift @@ -192,6 +192,25 @@ import Foundation /// The path to the GGUF model file. public let modelPath: String + /// The number of model layers to offload to the GPU. + /// + /// A negative value offloads all layers, and `0` runs entirely on the CPU. + public let gpuLayers: Int32 + + /// The default GPU layer count for the current platform. + /// + /// All layers are offloaded by default: the prebuilt llama.cpp binaries + /// ship with Metal enabled and the shader library embedded. The simulator + /// defaults to CPU-only execution, which remains the reliable + /// configuration there. + public static var defaultGPULayerCount: Int32 { + #if targetEnvironment(simulator) + return 0 + #else + return -1 + #endif + } + /// The context size for the model. /// /// - Important: This property is deprecated. @@ -425,8 +444,11 @@ import Foundation /// /// - Parameters: /// - modelPath: The path to the GGUF model file. - public init(modelPath: String) { + /// - gpuLayers: The number of model layers to offload to the GPU. + /// Defaults to ``defaultGPULayerCount``. + public init(modelPath: String, gpuLayers: Int32 = LlamaLanguageModel.defaultGPULayerCount) { self.modelPath = modelPath + self.gpuLayers = gpuLayers self.legacyDefaults = ResolvedGenerationOptions() } @@ -668,9 +690,7 @@ import Foundation private func createModelParams() -> llama_model_params { var params = llama_model_default_params() - - // Force CPU-only execution to avoid Metal GPU issues - params.n_gpu_layers = 0 + params.n_gpu_layers = gpuLayers // Try to reduce memory usage params.load_mode = LLAMA_LOAD_MODE_MMAP diff --git a/Tests/AnyLanguageModelTests/LlamaLanguageModelTests.swift b/Tests/AnyLanguageModelTests/LlamaLanguageModelTests.swift index 3d9e32c7..08efc898 100644 --- a/Tests/AnyLanguageModelTests/LlamaLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/LlamaLanguageModelTests.swift @@ -17,6 +17,10 @@ import Testing @Test func initialization() { let customModel = LlamaLanguageModel(modelPath: "/path/to/model.gguf") #expect(customModel.modelPath == "/path/to/model.gguf") + #expect(customModel.gpuLayers == LlamaLanguageModel.defaultGPULayerCount) + + let cpuOnlyModel = LlamaLanguageModel(modelPath: "/path/to/model.gguf", gpuLayers: 0) + #expect(cpuOnlyModel.gpuLayers == 0) #expect(customModel.contextSize == 2048) #expect(customModel.batchSize == 512) #expect(customModel.threads == Int32(ProcessInfo.processInfo.processorCount))