feat(viking): auto-return image bytes from viking_read for vision models

src/agentpool/agents/native_agent/agent.py

@@ -5,6 +5,7 @@
 import asyncio
 from collections.abc import Awaitable, Callable, Sequence
 from contextlib import AsyncExitStack, asynccontextmanager
+from dataclasses import replace
 from datetime import datetime, timedelta
 import inspect
 from pathlib import Path
@@ -1297,6 +1298,24 @@ async def get_agentlet[AgentOutputType](  # noqa: PLR0915
                             )
                             registry.register(populated, turn_scope)
 
+                # Populate VikingCapability.model_capabilities with resolved
+                # model capabilities so viking_read can auto-detect whether
+                # to return image bytes (via _should_return_image_bytes).
+                # Like ModalityFilterCapability this is a capability-level
+                # population, not auto-injection of new capabilities.
+                from agentpool.capabilities.viking import VikingCapability
+
+                if isinstance(cap, VikingCapability):
+                    populated_viking = replace(
+                        cap,
+                        model_capabilities=resolved_caps,
+                    )
+                    tool_capabilities[i] = populated_viking
+                    for j, ext_cap in enumerate(self._external_capabilities):
+                        if ext_cap is cap:
+                            self._external_capabilities[j] = populated_viking
+                            break
+
         # Handle retries parameter: newer pydantic-ai uses dict form for output_retries
         if AgentRetries is not None and self._output_retries is not None:
             retries_param: int | dict[str, int] = {

src/agentpool/capabilities/viking/__init__.py

@@ -26,6 +26,10 @@
 from pydantic_ai.capabilities import AbstractCapability
 from pydantic_ai.toolsets import AgentToolset, FunctionToolset
 
+from agentpool.capabilities.viking.constants import (
+    IMAGE_EXTENSIONS as IMAGE_EXTENSIONS,
+    IMAGE_MIME_TYPES as IMAGE_MIME_TYPES,
+)
 from agentpool.capabilities.viking.identity import VikingIdentity, _try_decode_api_key
 from agentpool.log import get_logger
 
@@ -90,6 +94,23 @@ class VikingCapability(AbstractCapability[Any]):
     multimodal_bridge: bool = False
     """Enable multimodal bridge — auto-upload binary content to Viking
     before sending to the model."""
+    support_vision: bool | None = None
+    """Result of ``viking_read`` for image URIs.
+
+    Tri-state control over how image resources are returned to the model:
+
+    - ``True`` — return image bytes (``BinaryImage``) regardless of model.
+    - ``False`` — return a text URI description, never image bytes.
+    - ``None`` (default) — auto-detect from ``model_capabilities.image_input``;
+      treated as text-only when capabilities are unknown (not injected or
+      field is ``None``).
+
+    Note: unlike ``ModalityFilterCapability._is_modality_supported``, which
+    treats ``capabilities=None`` as pass-through, this capability treats an
+    unset/model ``None`` capability as text-only (safe degradation) — it is
+    the *producer* of image content and must not emit ``BinaryImage`` it
+    cannot guarantee the model accepts.
+    """
     uploads_uri: str | None = None
     public_download_base_url: str | None = None
     enable_link: bool = False
@@ -1151,6 +1172,30 @@ async def _handle_multimodal_bridge(
             return request_context
         return replace(request_context, messages=new_messages)
 
+    def _should_return_image_bytes(self) -> bool:
+        """Whether ``viking_read`` should return image bytes for image URIs.
+
+        Decision order:
+
+        1. ``support_vision`` explicitly set — return its value.
+        2. ``model_capabilities`` injected — return ``image_input`` (``None``
+           counts as text-only).
+        3. Otherwise — text-only (safe degradation).
+
+        Note: unlike ``ModalityFilterCapability._is_modality_supported``
+        (which treats ``capabilities=None`` as pass-through), this treats an
+        unavailable capability as text-only: this capability *produces*
+        image content, so it must never emit ``BinaryImage`` it cannot
+        guarantee the model accepts.
+
+        Returns:
+            ``True`` when image bytes should be returned, ``False`` for text.
+        """
+        if self.support_vision is not None:
+            return self.support_vision
+        caps = self.model_capabilities
+        return bool(caps and caps.image_input)
+
     def _supports_modality(self, media_type: str) -> bool:
         """Check if the model supports the given media type.
 

src/agentpool/capabilities/viking/constants.py

@@ -0,0 +1,46 @@
+"""Constants for the Viking capability — image extension detection.
+
+Kept in a dedicated module (not ``__init__``) so ``tools.py`` can import
+them at runtime without importing the full capability package (avoiding
+import cycles, since the capability module lazily imports tools).
+"""
+
+from __future__ import annotations
+
+
+# Image extensions recognized by the openviking server parser layer
+# (``parse/parsers/media/constants.py``). MUST be kept in sync manually with
+# the server's ``IMAGE_EXTENSIONS`` — extension is the authoritative signal
+# since the server's ``stat`` API exposes no MIME field.
+#
+# ``.svg`` IS included (matching the server), but is a vector format most
+# vision APIs reject, so ``viking_read`` downgrades SVG URIs to a text hint
+# and never returns bytes for them. See ``_should_return_image_bytes``.
+IMAGE_EXTENSIONS: frozenset[str] = frozenset({
+    ".png",
+    ".jpg",
+    ".jpeg",
+    ".gif",
+    ".bmp",
+    ".webp",
+    ".svg",
+    ".tiff",
+    ".tif",
+    ".ico",
+    ".jp2",
+})
+
+# MIME mapping for the byte-return image extensions above. Unknown
+# extensions fall back to ``application/octet-stream``.
+IMAGE_MIME_TYPES: dict[str, str] = {
+    ".png": "image/png",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".gif": "image/gif",
+    ".bmp": "image/bmp",
+    ".webp": "image/webp",
+    ".tiff": "image/tiff",
+    ".tif": "image/tiff",
+    ".ico": "image/x-icon",
+    ".jp2": "image/jp2",
+}

src/agentpool/capabilities/viking/tools.py

@@ -9,12 +9,14 @@
 from __future__ import annotations
 
 import asyncio
+from pathlib import PurePosixPath
 from typing import TYPE_CHECKING, Any, Literal
 import uuid
 
-from pydantic_ai.messages import ToolReturn
+from pydantic_ai.messages import BinaryImage, ToolReturn
 from pydantic_ai.tools import RunContext  # noqa: TC002 - needed at runtime for get_type_hints()
 
+from agentpool.capabilities.viking.constants import IMAGE_EXTENSIONS, IMAGE_MIME_TYPES
 from agentpool.capabilities.viking.utils import (
     add_line_numbers,
     format_glob_results,
@@ -39,6 +41,29 @@ def _get_session_id(ctx: RunContext[Any]) -> str | None:
     return None
 
 
+def _is_image_resource(uri: str) -> bool:
+    """Whether a URI points to an image resource by its file extension.
+
+    Matches the openviking server's extension-based image detection
+    (``IMAGE_EXTENSIONS``). SVG is deliberately excluded — it is a vector
+    format most vision APIs reject, so it never enters the byte path.
+    """
+    return PurePosixPath(uri).suffix.lower() in IMAGE_EXTENSIONS
+
+
+def _image_uri_hint(uri: str) -> str:
+    """Text hint for an image URI when image bytes are not returned.
+
+    Used when the model cannot consume image bytes (text-only) or bytes
+    are forced off. Mentions the URI so the model can still reference it.
+    """
+    return (
+        f"[Image resource: {uri}]\n"
+        f"The file is an image and cannot be shown as text. The image is "
+        f"stored at the URI above — reference it when discussing the content."
+    )
+
+
 def build_tools(cap: VikingCapability) -> list[Callable[..., Any]]:
     """Build the list of tool functions for the Viking capability.
 
@@ -403,7 +428,36 @@ async def viking_read(
                 client = await cap._ensure_client()
                 uri_list = [uris] if isinstance(uris, str) else uris
                 sections: list[str] = []
+                image_parts: list[BinaryImage] = []
                 for u in uri_list:
+                    is_image = _is_image_resource(u)
+                    suffix = PurePosixPath(u).suffix.lower()
+                    # SVG is a vector format most vision APIs reject — it
+                    # never enters the byte path, always degrades to a text
+                    # hint, regardless of the support_vision / model caps.
+                    if is_image and (not cap._should_return_image_bytes() or suffix == ".svg"):
+                        # Image resource but the model can't consume image
+                        # bytes (or forced text / vector SVG) — text URI hint.
+                        if len(uri_list) > 1:
+                            sections.append(f"=== {u} ===\n{_image_uri_hint(u)}")
+                        else:
+                            sections.append(_image_uri_hint(u))
+                        continue
+
+                    if is_image:
+                        # Image resource and the model accepts image bytes.
+                        data = await client.download_bytes(u)
+                        media_type = IMAGE_MIME_TYPES.get(
+                            PurePosixPath(u).suffix.lower(), "application/octet-stream"
+                        )
+                        image_idx = len(image_parts) + 1  # 1-based, matches content order
+                        image_parts.append(BinaryImage(data=data, media_type=media_type))
+                        if len(uri_list) > 1:
+                            sections.append(f"=== {u} ===\n[Image #{image_idx}: {media_type}]")
+                        else:
+                            sections.append(f"[Image #{image_idx}: {media_type}]")
+                        continue
+
                     if level == "abstract":
                         content = await client.abstract(u)
                     elif level == "overview":
@@ -422,6 +476,16 @@ async def viking_read(
                         sections.append(f"=== {u} ===\n{numbered}")
                     else:
                         sections.append(numbered)
+
+                if image_parts:
+                    # Mixed content: text sections describe each file; image
+                    # bytes follow as BinaryImage parts the model can view.
+                    tool_content: list[Any] = ["\n\n".join(sections)]
+                    tool_content.extend(image_parts)
+                    return ToolReturn(
+                        return_value="\n\n".join(sections),
+                        content=tool_content,
+                    )
                 return ToolReturn(return_value="\n\n".join(sections))
             except Exception as e:
                 return ToolReturn(return_value=f"viking_read error: {e}")

src/agentpool_config/capabilities.py

@@ -184,6 +184,20 @@ class VikingCapabilityConfig(BaseModel):
     """Override for sessions URI. Default: viking://user/{user}/sessions/"""
     multimodal_bridge: bool = False
     """Enable multimodal bridge (Phase 6, not yet implemented)."""
+    support_vision: bool | None = None
+    """Result of viking_read for image URIs.
+
+    Tri-state control over how image resources are returned to the model:
+
+    - ``True`` — return image bytes (``BinaryImage``) regardless of model.
+    - ``False`` — return a text URI description, never image bytes.
+    - ``None`` (default) — auto-detect from resolved model capabilities
+      (``image_input``); text-only when unknown.
+
+    When forcing ``True`` on a model that does not actually accept image
+    input, configure ``type: modality_filter`` as a safety net so the
+    image is degraded before reaching the model API.
+    """
     uploads_uri: str | None = None
     """Override for uploads URI."""
     public_download_base_url: str | None = None