Add configurable VLM image preprocessing strategy

models/vlm/README.md

@@ -43,6 +43,31 @@ with asset roles consumed by the Swift runner's `ModelBundle`:
 Add a `VLMSpec(...)` entry to `SUPPORTED_MODELS` in
 [`vlm/export.py`](../../python/src/coreai_models/vlm/export.py) with the
 HuggingFace ID, output name, image token id, and vision geometry (resolution,
-patch/merge sizes, CLIP normalization stats). Models whose text decoder needs a
+patch/merge sizes, normalization stats). Models whose text decoder needs a
 new architecture also require a class registered in
 [`models/registry.py`](../../python/src/coreai_models/models/registry.py).
+
+## Image preprocessing
+
+The vision encoder expects a fixed-size square input. How an arbitrary image
+reaches that square is controlled by `image_strategy` in `metadata.json`:
+
+| Strategy      | Behavior                                 | Use when                             |
+|---------------|------------------------------------------|--------------------------------------|
+| `stretch`     | Resize directly to target size           | Default. Works for most models.      |
+| `center_crop` | Shortest-edge resize, then center crop   | CLIP-based vision towers (FastVLM)   |
+| `pad`         | Longest-edge resize, zero-pad remainder  | Models expecting preserved geometry  |
+
+The strategy is inferred from the model's `preprocessor_config.json` at export
+time and written into the bundle's `metadata.json`. Override at runtime:
+
+```bash
+llm-runner --model vlm_bundle --image photo.jpg --image-strategy center_crop
+```
+
+### Original resolution in prompt
+
+Some models (Qwen-VL family) benefit from knowing the original image
+dimensions. When `include_image_info` is set in the bundle metadata (or
+overridden via `--image-info on`), the original `W×H` is prepended to the
+text prompt before tokenization.

python/src/coreai_models/vlm/export.py

@@ -64,6 +64,8 @@ class VLMSpec:
     image_mean: tuple[float, float, float]
     image_std: tuple[float, float, float]
     rescale_factor: float
+    image_strategy: str = "stretch"
+    include_image_info: bool = False
 
     @property
     def num_visual_tokens(self) -> int:
@@ -84,6 +86,8 @@ def num_visual_tokens(self) -> int:
         image_mean=(0.5, 0.5, 0.5),
         image_std=(0.5, 0.5, 0.5),
         rescale_factor=1.0,
+        image_strategy="stretch",
+        include_image_info=True,
     ),
 }
 
@@ -378,6 +382,8 @@ async def export_text_bundle(
             "image_mean": list(spec.image_mean),
             "image_std": list(spec.image_std),
             "rescale_factor": spec.rescale_factor,
+            "image_strategy": spec.image_strategy,
+            "include_image_info": spec.include_image_info,
         },
         "source": {
             "hf_model_id": spec.hf_model_id,

swift/Sources/CoreAILanguageModels/Bundle/LanguageConfig.swift

@@ -169,6 +169,12 @@ public struct VisionConfig: Codable, Sendable, Equatable {
     /// Pixel rescale factor applied before normalization. Defaults to 1.0 when omitted.
     public let rescaleFactor: Double
 
+    /// Image preprocessing strategy. Defaults to stretch when omitted.
+    public let imageStrategy: ImageStrategy
+
+    /// Whether to include original image dimensions in the text prompt. Defaults to false.
+    public let includeImageInfo: Bool
+
     /// CLIP normalization (Qwen VL, Pixtral, InternVL, Phi-3.5-vision).
     public static let clipMean = [0.48145466, 0.4578275, 0.40821073]
     public static let clipStd = [0.26862954, 0.26130258, 0.27577711]
@@ -180,7 +186,9 @@ public struct VisionConfig: Codable, Sendable, Equatable {
         imageTokenId: Int32,
         imageMean: [Double]? = nil,
         imageStd: [Double]? = nil,
-        rescaleFactor: Double? = nil
+        rescaleFactor: Double? = nil,
+        imageStrategy: ImageStrategy? = nil,
+        includeImageInfo: Bool? = nil
     ) {
         self.imageSize = imageSize
         self.patchSize = patchSize
@@ -189,6 +197,8 @@ public struct VisionConfig: Codable, Sendable, Equatable {
         self.imageMean = imageMean ?? Self.clipMean
         self.imageStd = imageStd ?? Self.clipStd
         self.rescaleFactor = rescaleFactor ?? 1.0
+        self.imageStrategy = imageStrategy ?? .stretch
+        self.includeImageInfo = includeImageInfo ?? false
     }
 
     enum CodingKeys: String, CodingKey {
@@ -199,6 +209,8 @@ public struct VisionConfig: Codable, Sendable, Equatable {
         case imageMean = "image_mean"
         case imageStd = "image_std"
         case rescaleFactor = "rescale_factor"
+        case imageStrategy = "image_strategy"
+        case includeImageInfo = "include_image_info"
     }
 
     public init(from decoder: Swift.Decoder) throws {
@@ -210,5 +222,7 @@ public struct VisionConfig: Codable, Sendable, Equatable {
         self.imageMean = try c.decodeIfPresent([Double].self, forKey: .imageMean) ?? Self.clipMean
         self.imageStd = try c.decodeIfPresent([Double].self, forKey: .imageStd) ?? Self.clipStd
         self.rescaleFactor = try c.decodeIfPresent(Double.self, forKey: .rescaleFactor) ?? 1.0
+        self.imageStrategy = try c.decodeIfPresent(ImageStrategy.self, forKey: .imageStrategy) ?? .stretch
+        self.includeImageInfo = try c.decodeIfPresent(Bool.self, forKey: .includeImageInfo) ?? false
     }
 }

swift/Sources/CoreAILanguageModels/InferenceEngines/CoreAISequentialVLMEngine.swift

@@ -362,7 +362,8 @@ public final class CoreAISequentialVLMEngine: MultimodalInferenceEngine, @unchec
         )
 
         // Step 1: Preprocess image to CHW Float32
-        let chwPixels = try imagePreprocessor.preprocessCHW(cgImage: cgImage)
+        let chwPixels = try imagePreprocessor.preprocessCHW(
+            cgImage: cgImage, strategy: config.visionConfig.imageStrategy)
 
         // Step 2: Run encode_image
         let encoderOutput = try await runVisionEncoder(pixels: chwPixels)

swift/Sources/CoreAIShared/Image/ImagePreprocessor.swift

@@ -9,6 +9,14 @@ import CoreImage
 import Foundation
 import ImageIO
 
+// MARK: - Image Strategy
+
+public enum ImageStrategy: String, Codable, Sendable {
+    case stretch
+    case centerCrop = "center_crop"
+    case pad
+}
+
 // MARK: - ImagePreprocessor
 
 /// Resizes an image to a target size and applies per-channel normalization
@@ -92,7 +100,125 @@ public struct ImagePreprocessor: Sendable {
     public func preprocess(cgImage: CGImage) throws -> (Data, Int, Int) {
         let w = Int(targetSize.width)
         let h = Int(targetSize.height)
+        let resized = try renderToContext(
+            cgImage: cgImage, canvasWidth: w, canvasHeight: h,
+            drawRect: CGRect(x: 0, y: 0, width: w, height: h))
+        return try normalize(pixels: resized, width: w, height: h)
+    }
+
+    /// Preprocess a CGImage and return a flat CHW `[C, H, W]` Float32 array.
+    ///
+    /// Convenience wrapper over ``preprocess(cgImage:)`` that transposes the
+    /// NHWC RGBA output into the planar `[3, H, W]` layout expected by most
+    /// vision encoder inputs.
+    public func preprocessCHW(cgImage: CGImage) throws -> [Float] {
+        let (data, w, h) = try preprocess(cgImage: cgImage)
+        let pixelCount = w * h
+        var chw = [Float](repeating: 0, count: 3 * pixelCount)
+        data.withUnsafeBytes { rawSrc in
+            let src = rawSrc.bindMemory(to: Float.self)
+            for c in 0..<3 {
+                for i in 0..<pixelCount {
+                    chw[c * pixelCount + i] = src[i * 4 + c]
+                }
+            }
+        }
+        return chw
+    }
+
+    /// Preprocess with center-crop strategy: resize shortest edge to target,
+    /// then center-crop to square. Returns flat CHW `[3, H, W]` Float32.
+    public func preprocessCHWCenterCrop(cgImage: CGImage) throws -> [Float] {
+        let targetW = Int(targetSize.width)
+        let targetH = Int(targetSize.height)
+        let srcW = cgImage.width
+        let srcH = cgImage.height
+
+        let scale =
+            srcW < srcH
+            ? CGFloat(targetW) / CGFloat(srcW)
+            : CGFloat(targetH) / CGFloat(srcH)
+        let resizedW = Int(round(CGFloat(srcW) * scale))
+        let resizedH = Int(round(CGFloat(srcH) * scale))
+
+        let resized = try renderToContext(
+            cgImage: cgImage, canvasWidth: resizedW, canvasHeight: resizedH,
+            drawRect: CGRect(x: 0, y: 0, width: resizedW, height: resizedH))
+
+        let cropX = (resizedW - targetW) / 2
+        let cropY = (resizedH - targetH) / 2
+        guard let cropped = resized.cropping(to: CGRect(x: cropX, y: cropY, width: targetW, height: targetH)) else {
+            throw ImagePreprocessorError.renderFailed
+        }
+
+        return try preprocessCHW(cgImage: cropped)
+    }
+
+    /// Preprocess with pad strategy: resize longest edge to target, zero-pad
+    /// the shorter dimension. Returns flat CHW `[3, H, W]` Float32.
+    public func preprocessCHWPad(cgImage: CGImage) throws -> [Float] {
+        let targetW = Int(targetSize.width)
+        let targetH = Int(targetSize.height)
+        let srcW = cgImage.width
+        let srcH = cgImage.height
+
+        let scale =
+            srcW > srcH
+            ? CGFloat(targetW) / CGFloat(srcW)
+            : CGFloat(targetH) / CGFloat(srcH)
+        let resizedW = Int(round(CGFloat(srcW) * scale))
+        let resizedH = Int(round(CGFloat(srcH) * scale))
+
+        let offsetX = (targetW - resizedW) / 2
+        let offsetY = (targetH - resizedH) / 2
+
+        let padded = try renderToContext(
+            cgImage: cgImage, canvasWidth: targetW, canvasHeight: targetH,
+            drawRect: CGRect(x: offsetX, y: offsetY, width: resizedW, height: resizedH))
+
+        return try preprocessCHW(cgImage: padded)
+    }
+
+    /// Dispatch preprocessing based on strategy. Returns flat CHW `[3, H, W]`.
+    public func preprocessCHW(cgImage: CGImage, strategy: ImageStrategy) throws -> [Float] {
+        switch strategy {
+        case .stretch: return try preprocessCHW(cgImage: cgImage)
+        case .centerCrop: return try preprocessCHWCenterCrop(cgImage: cgImage)
+        case .pad: return try preprocessCHWPad(cgImage: cgImage)
+        }
+    }
 
+    // MARK: - Private Helpers
+
+    private func renderToContext(
+        cgImage: CGImage, canvasWidth: Int, canvasHeight: Int,
+        drawRect: CGRect
+    ) throws -> CGImage {
+        guard let colorSpace = CGColorSpace(name: CGColorSpace.sRGB) else {
+            throw ImagePreprocessorError.renderFailed
+        }
+        guard
+            let ctx = CGContext(
+                data: nil,
+                width: canvasWidth,
+                height: canvasHeight,
+                bitsPerComponent: 8,
+                bytesPerRow: canvasWidth * 4,
+                space: colorSpace,
+                bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue
+            )
+        else {
+            throw ImagePreprocessorError.renderFailed
+        }
+        ctx.interpolationQuality = .high
+        ctx.draw(cgImage, in: drawRect)
+        guard let result = ctx.makeImage() else {
+            throw ImagePreprocessorError.renderFailed
+        }
+        return result
+    }
+
+    private func normalize(pixels cgImage: CGImage, width w: Int, height h: Int) throws -> (Data, Int, Int) {
         guard let colorSpace = CGColorSpace(name: CGColorSpace.sRGB) else {
             throw ImagePreprocessorError.renderFailed
         }
@@ -109,7 +235,6 @@ public struct ImagePreprocessor: Sendable {
         else {
             throw ImagePreprocessorError.renderFailed
         }
-        ctx.interpolationQuality = .high
         ctx.draw(cgImage, in: CGRect(x: 0, y: 0, width: w, height: h))
 
         guard let pixelData = ctx.data else {
@@ -123,10 +248,6 @@ public struct ImagePreprocessor: Sendable {
         data.withUnsafeMutableBytes { dstPtr in
             guard let dstBase = dstPtr.bindMemory(to: Float.self).baseAddress else { return }
 
-            // Combine `(x*scale - mean) / std` into a single vDSP_vsmsa call:
-            //   y = x * (scale/std) + (-mean/std)
-            // For each color channel, read UInt8 with stride 4, write Float
-            // back to the interleaved RGBA destination at stride 4.
             let scale = Float(rescaleFactor) / 255.0
             let means: [Float] = [Float(mean.0), Float(mean.1), Float(mean.2)]
             let stds: [Float] = [Float(std.0), Float(std.1), Float(std.2)]
@@ -140,33 +261,12 @@ public struct ImagePreprocessor: Sendable {
                 vDSP_vsmsa(channel, 1, &a, &b, dstBase.advanced(by: c), 4, n)
             }
 
-            // Alpha unused — write 0 at stride 4.
             var zero: Float = 0
             vDSP_vfill(&zero, dstBase.advanced(by: 3), 4, n)
         }
 
         return (data, w, h)
     }
-
-    /// Preprocess a CGImage and return a flat CHW `[C, H, W]` Float32 array.
-    ///
-    /// Convenience wrapper over ``preprocess(cgImage:)`` that transposes the
-    /// NHWC RGBA output into the planar `[3, H, W]` layout expected by most
-    /// vision encoder inputs.
-    public func preprocessCHW(cgImage: CGImage) throws -> [Float] {
-        let (data, w, h) = try preprocess(cgImage: cgImage)
-        let pixelCount = w * h
-        var chw = [Float](repeating: 0, count: 3 * pixelCount)
-        data.withUnsafeBytes { rawSrc in
-            let src = rawSrc.bindMemory(to: Float.self)
-            for c in 0..<3 {
-                for i in 0..<pixelCount {
-                    chw[c * pixelCount + i] = src[i * 4 + c]
-                }
-            }
-        }
-        return chw
-    }
 }
 
 // MARK: - Errors

swift/Sources/Tools/llm-runner/LLMRunnerMain.swift

@@ -7,6 +7,7 @@ import ArgumentParser
 import CoreAI
 import CoreAILanguageModels
 import CoreAIShared
+import CoreImage
 import Darwin
 import Foundation
 import Tokenizers
@@ -41,6 +42,14 @@ enum WarmupMode: ExpressibleByArgument, Equatable {
     }
 }
 
+extension ImageStrategy: ExpressibleByArgument {}
+
+enum ImageInfoMode: String, ExpressibleByArgument, Sendable {
+    case on
+    case off
+    case auto
+}
+
 @main
 struct Main {
     static func main() async throws {
@@ -179,6 +188,16 @@ struct LLMRunner: AsyncParsableCommand, Sendable {
     @Option(name: .customLong("image"), help: "Path to an image file for vision-language models")
     var imagePath: String?
 
+    @Option(
+        name: .customLong("image-strategy"),
+        help: "Image preprocessing: stretch, center_crop, or pad (default: from model metadata)")
+    var imageStrategy: ImageStrategy?
+
+    @Option(
+        name: .customLong("image-info"),
+        help: "Include original image resolution in prompt: on, off, auto (default: auto)")
+    var imageInfo: ImageInfoMode = .auto
+
     @Flag(help: "Enable verbose logging")
     var verbose: Bool = false
 
@@ -839,6 +858,24 @@ struct LLMRunner: AsyncParsableCommand, Sendable {
         let embeddedInput = try await vlmEngine.encodeImage(at: imageURL)
         CLILogger.log("Image encoded: \(embeddedInput.tokenCount) visual tokens", component: "VLM")
 
+        let shouldIncludeInfo =
+            switch imageInfo {
+            case .on: true
+            case .off: false
+            case .auto: visionConfig.includeImageInfo
+            }
+
+        var effectivePrompt = displayPrompt
+        if shouldIncludeInfo {
+            let imageURL = URL(fileURLWithPath: imagePath)
+            if let ciImage = CIImage(contentsOf: imageURL) {
+                let w = Int(ciImage.extent.width)
+                let h = Int(ciImage.extent.height)
+                effectivePrompt = "Image: \(w)x\(h)\n\(displayPrompt)"
+                CLILogger.log("Injected image info: \(w)x\(h)", component: "VLM")
+            }
+        }
+
         // Build VLM prompt with image placeholder tokens.
         // Try using the tokenizer's chat template if available; fall back to
         // generic "USER: <image>×N \n prompt \nASSISTANT:" format.
@@ -847,7 +884,7 @@ struct LLMRunner: AsyncParsableCommand, Sendable {
         let vlmTokens: [Int32]
 
         if let chatTemplateTokens = try? buildVLMPromptFromChatTemplate(
-            prompt: displayPrompt,
+            prompt: effectivePrompt,
             imageTokenCount: imageTokenCount,
             imageTokenId: imageTokenId,
             tokenizer: tokenizer
@@ -860,7 +897,7 @@ struct LLMRunner: AsyncParsableCommand, Sendable {
                 component: "VLM")
             var tokens = tokenizer.encode(text: "USER: ", addSpecialTokens: true).map { Int32($0) }
             tokens.append(contentsOf: [Int32](repeating: imageTokenId, count: imageTokenCount))
-            let suffix = "\n" + displayPrompt + "\nASSISTANT:"
+            let suffix = "\n" + effectivePrompt + "\nASSISTANT:"
             tokens.append(
                 contentsOf: tokenizer.encode(text: suffix, addSpecialTokens: false).map { Int32($0) })
             vlmTokens = tokens