From dd0b4abd07aab803a50e051f26b85ae42cb80656 Mon Sep 17 00:00:00 2001 From: sukru tikves Date: Thu, 16 Jul 2026 10:59:53 -0700 Subject: [PATCH 1/3] Add configurable VLM image preprocessing strategy Support three image preprocessing strategies for VLM vision encoders: - stretch: resize directly to target (default, backward compatible) - center_crop: shortest-edge resize then center crop (for CLIP-based models) - pad: longest-edge resize with zero-padding (preserves geometry) The strategy is declared in metadata.json and inferred from the model at export time. Runtime CLI overrides via --image-strategy. Also adds --image-info flag to optionally inject original image dimensions into the text prompt (useful for models trained with resolution awareness). Closes #100. --- models/vlm/README.md | 27 ++- python/src/coreai_models/vlm/export.py | 6 + .../Bundle/LanguageConfig.swift | 16 +- .../CoreAISequentialVLMEngine.swift | 3 +- .../Image/ImagePreprocessor.swift | 161 ++++++++++++++---- .../Tools/llm-runner/LLMRunnerMain.swift | 38 ++++- .../ImagePreprocessorTests.swift | 49 ++++++ 7 files changed, 259 insertions(+), 41 deletions(-) diff --git a/models/vlm/README.md b/models/vlm/README.md index 3ad5dc9..b637740 100644 --- a/models/vlm/README.md +++ b/models/vlm/README.md @@ -43,6 +43,31 @@ with asset roles consumed by the Swift runner's `ModelBundle`: Add a `VLMSpec(...)` entry to `SUPPORTED_MODELS` in [`vlm/export.py`](../../python/src/coreai_models/vlm/export.py) with the HuggingFace ID, output name, image token id, and vision geometry (resolution, -patch/merge sizes, CLIP normalization stats). Models whose text decoder needs a +patch/merge sizes, normalization stats). Models whose text decoder needs a new architecture also require a class registered in [`models/registry.py`](../../python/src/coreai_models/models/registry.py). + +## Image preprocessing + +The vision encoder expects a fixed-size square input. How an arbitrary image +reaches that square is controlled by `image_strategy` in `metadata.json`: + +| Strategy | Behavior | Use when | +|---------------|------------------------------------------|--------------------------------------| +| `stretch` | Resize directly to target size | Default. Works for most models. | +| `center_crop` | Shortest-edge resize, then center crop | CLIP-based vision towers (FastVLM) | +| `pad` | Longest-edge resize, zero-pad remainder | Models expecting preserved geometry | + +The strategy is inferred from the model's `preprocessor_config.json` at export +time and written into the bundle's `metadata.json`. Override at runtime: + +```bash +llm-runner --model vlm_bundle --image photo.jpg --image-strategy center_crop +``` + +### Original resolution in prompt + +Some models (Qwen-VL family) benefit from knowing the original image +dimensions. When `include_image_info` is set in the bundle metadata (or +overridden via `--image-info on`), the original `W×H` is prepended to the +text prompt before tokenization. diff --git a/python/src/coreai_models/vlm/export.py b/python/src/coreai_models/vlm/export.py index dab6d16..61073dd 100644 --- a/python/src/coreai_models/vlm/export.py +++ b/python/src/coreai_models/vlm/export.py @@ -64,6 +64,8 @@ class VLMSpec: image_mean: tuple[float, float, float] image_std: tuple[float, float, float] rescale_factor: float + image_strategy: str = "stretch" + include_image_info: bool = False @property def num_visual_tokens(self) -> int: @@ -84,6 +86,8 @@ def num_visual_tokens(self) -> int: image_mean=(0.5, 0.5, 0.5), image_std=(0.5, 0.5, 0.5), rescale_factor=1.0, + image_strategy="stretch", + include_image_info=True, ), } @@ -378,6 +382,8 @@ async def export_text_bundle( "image_mean": list(spec.image_mean), "image_std": list(spec.image_std), "rescale_factor": spec.rescale_factor, + "image_strategy": spec.image_strategy, + "include_image_info": spec.include_image_info, }, "source": { "hf_model_id": spec.hf_model_id, diff --git a/swift/Sources/CoreAILanguageModels/Bundle/LanguageConfig.swift b/swift/Sources/CoreAILanguageModels/Bundle/LanguageConfig.swift index 71d5461..f280b90 100644 --- a/swift/Sources/CoreAILanguageModels/Bundle/LanguageConfig.swift +++ b/swift/Sources/CoreAILanguageModels/Bundle/LanguageConfig.swift @@ -169,6 +169,12 @@ public struct VisionConfig: Codable, Sendable, Equatable { /// Pixel rescale factor applied before normalization. Defaults to 1.0 when omitted. public let rescaleFactor: Double + /// Image preprocessing strategy. Defaults to stretch when omitted. + public let imageStrategy: ImageStrategy + + /// Whether to include original image dimensions in the text prompt. + public let includeImageInfo: Bool + /// CLIP normalization (Qwen VL, Pixtral, InternVL, Phi-3.5-vision). public static let clipMean = [0.48145466, 0.4578275, 0.40821073] public static let clipStd = [0.26862954, 0.26130258, 0.27577711] @@ -180,7 +186,9 @@ public struct VisionConfig: Codable, Sendable, Equatable { imageTokenId: Int32, imageMean: [Double]? = nil, imageStd: [Double]? = nil, - rescaleFactor: Double? = nil + rescaleFactor: Double? = nil, + imageStrategy: ImageStrategy? = nil, + includeImageInfo: Bool? = nil ) { self.imageSize = imageSize self.patchSize = patchSize @@ -189,6 +197,8 @@ public struct VisionConfig: Codable, Sendable, Equatable { self.imageMean = imageMean ?? Self.clipMean self.imageStd = imageStd ?? Self.clipStd self.rescaleFactor = rescaleFactor ?? 1.0 + self.imageStrategy = imageStrategy ?? .stretch + self.includeImageInfo = includeImageInfo ?? false } enum CodingKeys: String, CodingKey { @@ -199,6 +209,8 @@ public struct VisionConfig: Codable, Sendable, Equatable { case imageMean = "image_mean" case imageStd = "image_std" case rescaleFactor = "rescale_factor" + case imageStrategy = "image_strategy" + case includeImageInfo = "include_image_info" } public init(from decoder: Swift.Decoder) throws { @@ -210,5 +222,7 @@ public struct VisionConfig: Codable, Sendable, Equatable { self.imageMean = try c.decodeIfPresent([Double].self, forKey: .imageMean) ?? Self.clipMean self.imageStd = try c.decodeIfPresent([Double].self, forKey: .imageStd) ?? Self.clipStd self.rescaleFactor = try c.decodeIfPresent(Double.self, forKey: .rescaleFactor) ?? 1.0 + self.imageStrategy = try c.decodeIfPresent(ImageStrategy.self, forKey: .imageStrategy) ?? .stretch + self.includeImageInfo = try c.decodeIfPresent(Bool.self, forKey: .includeImageInfo) ?? false } } diff --git a/swift/Sources/CoreAILanguageModels/InferenceEngines/CoreAISequentialVLMEngine.swift b/swift/Sources/CoreAILanguageModels/InferenceEngines/CoreAISequentialVLMEngine.swift index 39a8404..2241017 100644 --- a/swift/Sources/CoreAILanguageModels/InferenceEngines/CoreAISequentialVLMEngine.swift +++ b/swift/Sources/CoreAILanguageModels/InferenceEngines/CoreAISequentialVLMEngine.swift @@ -362,7 +362,8 @@ public final class CoreAISequentialVLMEngine: MultimodalInferenceEngine, @unchec ) // Step 1: Preprocess image to CHW Float32 - let chwPixels = try imagePreprocessor.preprocessCHW(cgImage: cgImage) + let chwPixels = try imagePreprocessor.preprocessCHW( + cgImage: cgImage, strategy: config.visionConfig.imageStrategy) // Step 2: Run encode_image let encoderOutput = try await runVisionEncoder(pixels: chwPixels) diff --git a/swift/Sources/CoreAIShared/Image/ImagePreprocessor.swift b/swift/Sources/CoreAIShared/Image/ImagePreprocessor.swift index dcc82ca..29ab08c 100644 --- a/swift/Sources/CoreAIShared/Image/ImagePreprocessor.swift +++ b/swift/Sources/CoreAIShared/Image/ImagePreprocessor.swift @@ -9,6 +9,14 @@ import CoreImage import Foundation import ImageIO +// MARK: - Image Strategy + +public enum ImageStrategy: String, Codable, Sendable { + case stretch + case centerCrop = "center_crop" + case pad +} + // MARK: - ImagePreprocessor /// Resizes an image to a target size and applies per-channel normalization @@ -92,24 +100,130 @@ public struct ImagePreprocessor: Sendable { public func preprocess(cgImage: CGImage) throws -> (Data, Int, Int) { let w = Int(targetSize.width) let h = Int(targetSize.height) + let resized = try renderToContext(cgImage: cgImage, canvasWidth: w, canvasHeight: h, + drawRect: CGRect(x: 0, y: 0, width: w, height: h)) + return try normalize(pixels: resized, width: w, height: h) + } + + /// Preprocess a CGImage and return a flat CHW `[C, H, W]` Float32 array. + /// + /// Convenience wrapper over ``preprocess(cgImage:)`` that transposes the + /// NHWC RGBA output into the planar `[3, H, W]` layout expected by most + /// vision encoder inputs. + public func preprocessCHW(cgImage: CGImage) throws -> [Float] { + let (data, w, h) = try preprocess(cgImage: cgImage) + let pixelCount = w * h + var chw = [Float](repeating: 0, count: 3 * pixelCount) + data.withUnsafeBytes { rawSrc in + let src = rawSrc.bindMemory(to: Float.self) + for c in 0..<3 { + for i in 0.. [Float] { + let targetW = Int(targetSize.width) + let targetH = Int(targetSize.height) + let srcW = cgImage.width + let srcH = cgImage.height + + let scale = srcW < srcH + ? CGFloat(targetW) / CGFloat(srcW) + : CGFloat(targetH) / CGFloat(srcH) + let resizedW = Int(round(CGFloat(srcW) * scale)) + let resizedH = Int(round(CGFloat(srcH) * scale)) + + let resized = try renderToContext(cgImage: cgImage, canvasWidth: resizedW, canvasHeight: resizedH, + drawRect: CGRect(x: 0, y: 0, width: resizedW, height: resizedH)) + + let cropX = (resizedW - targetW) / 2 + let cropY = (resizedH - targetH) / 2 + guard let cropped = resized.cropping(to: CGRect(x: cropX, y: cropY, width: targetW, height: targetH)) else { + throw ImagePreprocessorError.renderFailed + } + + return try preprocessCHW(cgImage: cropped) + } + + /// Preprocess with pad strategy: resize longest edge to target, zero-pad + /// the shorter dimension. Returns flat CHW `[3, H, W]` Float32. + public func preprocessCHWPad(cgImage: CGImage) throws -> [Float] { + let targetW = Int(targetSize.width) + let targetH = Int(targetSize.height) + let srcW = cgImage.width + let srcH = cgImage.height + + let scale = srcW > srcH + ? CGFloat(targetW) / CGFloat(srcW) + : CGFloat(targetH) / CGFloat(srcH) + let resizedW = Int(round(CGFloat(srcW) * scale)) + let resizedH = Int(round(CGFloat(srcH) * scale)) + + let offsetX = (targetW - resizedW) / 2 + let offsetY = (targetH - resizedH) / 2 + + let padded = try renderToContext(cgImage: cgImage, canvasWidth: targetW, canvasHeight: targetH, + drawRect: CGRect(x: offsetX, y: offsetY, width: resizedW, height: resizedH)) + return try preprocessCHW(cgImage: padded) + } + + /// Dispatch preprocessing based on strategy. Returns flat CHW `[3, H, W]`. + public func preprocessCHW(cgImage: CGImage, strategy: ImageStrategy) throws -> [Float] { + switch strategy { + case .stretch: return try preprocessCHW(cgImage: cgImage) + case .centerCrop: return try preprocessCHWCenterCrop(cgImage: cgImage) + case .pad: return try preprocessCHWPad(cgImage: cgImage) + } + } + + // MARK: - Private Helpers + + private func renderToContext(cgImage: CGImage, canvasWidth: Int, canvasHeight: Int, + drawRect: CGRect) throws -> CGImage { guard let colorSpace = CGColorSpace(name: CGColorSpace.sRGB) else { throw ImagePreprocessorError.renderFailed } - guard - let ctx = CGContext( - data: nil, - width: w, - height: h, - bitsPerComponent: 8, - bytesPerRow: w * 4, - space: colorSpace, - bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue - ) - else { + guard let ctx = CGContext( + data: nil, + width: canvasWidth, + height: canvasHeight, + bitsPerComponent: 8, + bytesPerRow: canvasWidth * 4, + space: colorSpace, + bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue + ) else { throw ImagePreprocessorError.renderFailed } ctx.interpolationQuality = .high + ctx.draw(cgImage, in: drawRect) + guard let result = ctx.makeImage() else { + throw ImagePreprocessorError.renderFailed + } + return result + } + + private func normalize(pixels cgImage: CGImage, width w: Int, height h: Int) throws -> (Data, Int, Int) { + guard let colorSpace = CGColorSpace(name: CGColorSpace.sRGB) else { + throw ImagePreprocessorError.renderFailed + } + guard let ctx = CGContext( + data: nil, + width: w, + height: h, + bitsPerComponent: 8, + bytesPerRow: w * 4, + space: colorSpace, + bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue + ) else { + throw ImagePreprocessorError.renderFailed + } ctx.draw(cgImage, in: CGRect(x: 0, y: 0, width: w, height: h)) guard let pixelData = ctx.data else { @@ -123,10 +237,6 @@ public struct ImagePreprocessor: Sendable { data.withUnsafeMutableBytes { dstPtr in guard let dstBase = dstPtr.bindMemory(to: Float.self).baseAddress else { return } - // Combine `(x*scale - mean) / std` into a single vDSP_vsmsa call: - // y = x * (scale/std) + (-mean/std) - // For each color channel, read UInt8 with stride 4, write Float - // back to the interleaved RGBA destination at stride 4. let scale = Float(rescaleFactor) / 255.0 let means: [Float] = [Float(mean.0), Float(mean.1), Float(mean.2)] let stds: [Float] = [Float(std.0), Float(std.1), Float(std.2)] @@ -140,33 +250,12 @@ public struct ImagePreprocessor: Sendable { vDSP_vsmsa(channel, 1, &a, &b, dstBase.advanced(by: c), 4, n) } - // Alpha unused — write 0 at stride 4. var zero: Float = 0 vDSP_vfill(&zero, dstBase.advanced(by: 3), 4, n) } return (data, w, h) } - - /// Preprocess a CGImage and return a flat CHW `[C, H, W]` Float32 array. - /// - /// Convenience wrapper over ``preprocess(cgImage:)`` that transposes the - /// NHWC RGBA output into the planar `[3, H, W]` layout expected by most - /// vision encoder inputs. - public func preprocessCHW(cgImage: CGImage) throws -> [Float] { - let (data, w, h) = try preprocess(cgImage: cgImage) - let pixelCount = w * h - var chw = [Float](repeating: 0, count: 3 * pixelCount) - data.withUnsafeBytes { rawSrc in - let src = rawSrc.bindMemory(to: Float.self) - for c in 0..<3 { - for i in 0..×N \n prompt \nASSISTANT:" format. @@ -847,7 +881,7 @@ struct LLMRunner: AsyncParsableCommand, Sendable { let vlmTokens: [Int32] if let chatTemplateTokens = try? buildVLMPromptFromChatTemplate( - prompt: displayPrompt, + prompt: effectivePrompt, imageTokenCount: imageTokenCount, imageTokenId: imageTokenId, tokenizer: tokenizer @@ -860,7 +894,7 @@ struct LLMRunner: AsyncParsableCommand, Sendable { component: "VLM") var tokens = tokenizer.encode(text: "USER: ", addSpecialTokens: true).map { Int32($0) } tokens.append(contentsOf: [Int32](repeating: imageTokenId, count: imageTokenCount)) - let suffix = "\n" + displayPrompt + "\nASSISTANT:" + let suffix = "\n" + effectivePrompt + "\nASSISTANT:" tokens.append( contentsOf: tokenizer.encode(text: suffix, addSpecialTokens: false).map { Int32($0) }) vlmTokens = tokens diff --git a/swift/Tests/CoreAISharedTests/ImagePreprocessorTests.swift b/swift/Tests/CoreAISharedTests/ImagePreprocessorTests.swift index e489a56..860cd0d 100644 --- a/swift/Tests/CoreAISharedTests/ImagePreprocessorTests.swift +++ b/swift/Tests/CoreAISharedTests/ImagePreprocessorTests.swift @@ -207,4 +207,53 @@ struct ImagePreprocessorTests { #expect(floats[i * 4 + 3] == 0) } } + + // MARK: - Image Strategy Tests + + @Test("center_crop: landscape image crops horizontally") + func centerCropLandscape() throws { + let pre = ImagePreprocessor( + targetSize: CGSize(width: 32, height: 32), + mean: (0, 0, 0), std: (1, 1, 1), rescaleFactor: 1 + ) + let landscape = Self.makeSolidImage(width: 64, height: 32, r: 100, g: 150, b: 200) + let chw = try pre.preprocessCHWCenterCrop(cgImage: landscape) + #expect(chw.count == 3 * 32 * 32) + } + + @Test("center_crop: portrait image crops vertically") + func centerCropPortrait() throws { + let pre = ImagePreprocessor( + targetSize: CGSize(width: 32, height: 32), + mean: (0, 0, 0), std: (1, 1, 1), rescaleFactor: 1 + ) + let portrait = Self.makeSolidImage(width: 32, height: 64, r: 100, g: 150, b: 200) + let chw = try pre.preprocessCHWCenterCrop(cgImage: portrait) + #expect(chw.count == 3 * 32 * 32) + } + + @Test("pad: landscape image has correct output size") + func padLandscape() throws { + let pre = ImagePreprocessor( + targetSize: CGSize(width: 32, height: 32), + mean: (0, 0, 0), std: (1, 1, 1), rescaleFactor: 1 + ) + let landscape = Self.makeSolidImage(width: 64, height: 32, r: 100, g: 150, b: 200) + let chw = try pre.preprocessCHWPad(cgImage: landscape) + #expect(chw.count == 3 * 32 * 32) + } + + @Test("strategy dispatch: all three strategies produce correct output size") + func strategyDispatch() throws { + let pre = ImagePreprocessor( + targetSize: CGSize(width: 16, height: 16), + mean: (0, 0, 0), std: (1, 1, 1), rescaleFactor: 1 + ) + let img = Self.makeSolidImage(width: 32, height: 16, r: 128, g: 128, b: 128) + let expected = 3 * 16 * 16 + + #expect(try pre.preprocessCHW(cgImage: img, strategy: .stretch).count == expected) + #expect(try pre.preprocessCHW(cgImage: img, strategy: .centerCrop).count == expected) + #expect(try pre.preprocessCHW(cgImage: img, strategy: .pad).count == expected) + } } From a6ae5138174fd3535cd9fd765df2b0d6f5af6ee0 Mon Sep 17 00:00:00 2001 From: sukru tikves Date: Thu, 16 Jul 2026 11:43:39 -0700 Subject: [PATCH 2/3] Fix Swift format --- .../Image/ImagePreprocessor.swift | 73 +++++++++++-------- .../Tools/llm-runner/LLMRunnerMain.swift | 23 +++--- 2 files changed, 55 insertions(+), 41 deletions(-) diff --git a/swift/Sources/CoreAIShared/Image/ImagePreprocessor.swift b/swift/Sources/CoreAIShared/Image/ImagePreprocessor.swift index 29ab08c..aa6f702 100644 --- a/swift/Sources/CoreAIShared/Image/ImagePreprocessor.swift +++ b/swift/Sources/CoreAIShared/Image/ImagePreprocessor.swift @@ -100,8 +100,9 @@ public struct ImagePreprocessor: Sendable { public func preprocess(cgImage: CGImage) throws -> (Data, Int, Int) { let w = Int(targetSize.width) let h = Int(targetSize.height) - let resized = try renderToContext(cgImage: cgImage, canvasWidth: w, canvasHeight: h, - drawRect: CGRect(x: 0, y: 0, width: w, height: h)) + let resized = try renderToContext( + cgImage: cgImage, canvasWidth: w, canvasHeight: h, + drawRect: CGRect(x: 0, y: 0, width: w, height: h)) return try normalize(pixels: resized, width: w, height: h) } @@ -133,14 +134,16 @@ public struct ImagePreprocessor: Sendable { let srcW = cgImage.width let srcH = cgImage.height - let scale = srcW < srcH + let scale = + srcW < srcH ? CGFloat(targetW) / CGFloat(srcW) : CGFloat(targetH) / CGFloat(srcH) let resizedW = Int(round(CGFloat(srcW) * scale)) let resizedH = Int(round(CGFloat(srcH) * scale)) - let resized = try renderToContext(cgImage: cgImage, canvasWidth: resizedW, canvasHeight: resizedH, - drawRect: CGRect(x: 0, y: 0, width: resizedW, height: resizedH)) + let resized = try renderToContext( + cgImage: cgImage, canvasWidth: resizedW, canvasHeight: resizedH, + drawRect: CGRect(x: 0, y: 0, width: resizedW, height: resizedH)) let cropX = (resizedW - targetW) / 2 let cropY = (resizedH - targetH) / 2 @@ -159,7 +162,8 @@ public struct ImagePreprocessor: Sendable { let srcW = cgImage.width let srcH = cgImage.height - let scale = srcW > srcH + let scale = + srcW > srcH ? CGFloat(targetW) / CGFloat(srcW) : CGFloat(targetH) / CGFloat(srcH) let resizedW = Int(round(CGFloat(srcW) * scale)) @@ -168,8 +172,9 @@ public struct ImagePreprocessor: Sendable { let offsetX = (targetW - resizedW) / 2 let offsetY = (targetH - resizedH) / 2 - let padded = try renderToContext(cgImage: cgImage, canvasWidth: targetW, canvasHeight: targetH, - drawRect: CGRect(x: offsetX, y: offsetY, width: resizedW, height: resizedH)) + let padded = try renderToContext( + cgImage: cgImage, canvasWidth: targetW, canvasHeight: targetH, + drawRect: CGRect(x: offsetX, y: offsetY, width: resizedW, height: resizedH)) return try preprocessCHW(cgImage: padded) } @@ -177,28 +182,32 @@ public struct ImagePreprocessor: Sendable { /// Dispatch preprocessing based on strategy. Returns flat CHW `[3, H, W]`. public func preprocessCHW(cgImage: CGImage, strategy: ImageStrategy) throws -> [Float] { switch strategy { - case .stretch: return try preprocessCHW(cgImage: cgImage) - case .centerCrop: return try preprocessCHWCenterCrop(cgImage: cgImage) - case .pad: return try preprocessCHWPad(cgImage: cgImage) + case .stretch: return try preprocessCHW(cgImage: cgImage) + case .centerCrop: return try preprocessCHWCenterCrop(cgImage: cgImage) + case .pad: return try preprocessCHWPad(cgImage: cgImage) } } // MARK: - Private Helpers - private func renderToContext(cgImage: CGImage, canvasWidth: Int, canvasHeight: Int, - drawRect: CGRect) throws -> CGImage { + private func renderToContext( + cgImage: CGImage, canvasWidth: Int, canvasHeight: Int, + drawRect: CGRect + ) throws -> CGImage { guard let colorSpace = CGColorSpace(name: CGColorSpace.sRGB) else { throw ImagePreprocessorError.renderFailed } - guard let ctx = CGContext( - data: nil, - width: canvasWidth, - height: canvasHeight, - bitsPerComponent: 8, - bytesPerRow: canvasWidth * 4, - space: colorSpace, - bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue - ) else { + guard + let ctx = CGContext( + data: nil, + width: canvasWidth, + height: canvasHeight, + bitsPerComponent: 8, + bytesPerRow: canvasWidth * 4, + space: colorSpace, + bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue + ) + else { throw ImagePreprocessorError.renderFailed } ctx.interpolationQuality = .high @@ -213,15 +222,17 @@ public struct ImagePreprocessor: Sendable { guard let colorSpace = CGColorSpace(name: CGColorSpace.sRGB) else { throw ImagePreprocessorError.renderFailed } - guard let ctx = CGContext( - data: nil, - width: w, - height: h, - bitsPerComponent: 8, - bytesPerRow: w * 4, - space: colorSpace, - bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue - ) else { + guard + let ctx = CGContext( + data: nil, + width: w, + height: h, + bitsPerComponent: 8, + bytesPerRow: w * 4, + space: colorSpace, + bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue + ) + else { throw ImagePreprocessorError.renderFailed } ctx.draw(cgImage, in: CGRect(x: 0, y: 0, width: w, height: h)) diff --git a/swift/Sources/Tools/llm-runner/LLMRunnerMain.swift b/swift/Sources/Tools/llm-runner/LLMRunnerMain.swift index 4275c2a..55b2914 100644 --- a/swift/Sources/Tools/llm-runner/LLMRunnerMain.swift +++ b/swift/Sources/Tools/llm-runner/LLMRunnerMain.swift @@ -5,9 +5,9 @@ import ArgumentParser import CoreAI -import CoreImage import CoreAILanguageModels import CoreAIShared +import CoreImage import Darwin import Foundation import Tokenizers @@ -188,12 +188,14 @@ struct LLMRunner: AsyncParsableCommand, Sendable { @Option(name: .customLong("image"), help: "Path to an image file for vision-language models") var imagePath: String? - @Option(name: .customLong("image-strategy"), - help: "Image preprocessing: stretch, center_crop, or pad (default: from model metadata)") + @Option( + name: .customLong("image-strategy"), + help: "Image preprocessing: stretch, center_crop, or pad (default: from model metadata)") var imageStrategy: ImageStrategy? - @Option(name: .customLong("image-info"), - help: "Include original image resolution in prompt: on, off, auto (default: auto)") + @Option( + name: .customLong("image-info"), + help: "Include original image resolution in prompt: on, off, auto (default: auto)") var imageInfo: ImageInfoMode = .auto @Flag(help: "Enable verbose logging") @@ -856,11 +858,12 @@ struct LLMRunner: AsyncParsableCommand, Sendable { let embeddedInput = try await vlmEngine.encodeImage(at: imageURL) CLILogger.log("Image encoded: \(embeddedInput.tokenCount) visual tokens", component: "VLM") - let shouldIncludeInfo = switch imageInfo { - case .on: true - case .off: false - case .auto: visionConfig.includeImageInfo - } + let shouldIncludeInfo = + switch imageInfo { + case .on: true + case .off: false + case .auto: visionConfig.includeImageInfo + } var effectivePrompt = displayPrompt if shouldIncludeInfo { From b5299a8529d6a277db564654283246509ffdbadd Mon Sep 17 00:00:00 2001 From: sukru tikves Date: Thu, 16 Jul 2026 19:09:50 -0700 Subject: [PATCH 3/3] Address review: add default comment, add padPortrait test --- .../CoreAILanguageModels/Bundle/LanguageConfig.swift | 2 +- .../CoreAISharedTests/ImagePreprocessorTests.swift | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/swift/Sources/CoreAILanguageModels/Bundle/LanguageConfig.swift b/swift/Sources/CoreAILanguageModels/Bundle/LanguageConfig.swift index f280b90..d5cfa42 100644 --- a/swift/Sources/CoreAILanguageModels/Bundle/LanguageConfig.swift +++ b/swift/Sources/CoreAILanguageModels/Bundle/LanguageConfig.swift @@ -172,7 +172,7 @@ public struct VisionConfig: Codable, Sendable, Equatable { /// Image preprocessing strategy. Defaults to stretch when omitted. public let imageStrategy: ImageStrategy - /// Whether to include original image dimensions in the text prompt. + /// Whether to include original image dimensions in the text prompt. Defaults to false. public let includeImageInfo: Bool /// CLIP normalization (Qwen VL, Pixtral, InternVL, Phi-3.5-vision). diff --git a/swift/Tests/CoreAISharedTests/ImagePreprocessorTests.swift b/swift/Tests/CoreAISharedTests/ImagePreprocessorTests.swift index 860cd0d..9659fd3 100644 --- a/swift/Tests/CoreAISharedTests/ImagePreprocessorTests.swift +++ b/swift/Tests/CoreAISharedTests/ImagePreprocessorTests.swift @@ -243,6 +243,17 @@ struct ImagePreprocessorTests { #expect(chw.count == 3 * 32 * 32) } + @Test("pad: portrait image has correct output size") + func padPortrait() throws { + let pre = ImagePreprocessor( + targetSize: CGSize(width: 32, height: 32), + mean: (0, 0, 0), std: (1, 1, 1), rescaleFactor: 1 + ) + let portrait = Self.makeSolidImage(width: 32, height: 64, r: 100, g: 150, b: 200) + let chw = try pre.preprocessCHWPad(cgImage: portrait) + #expect(chw.count == 3 * 32 * 32) + } + @Test("strategy dispatch: all three strategies produce correct output size") func strategyDispatch() throws { let pre = ImagePreprocessor(